mirror of
https://github.com/RipMeApp/ripme.git
synced 2025-08-16 10:54:09 +02:00
Refactor, removed duplicated code. Fixed bugs
This commit is contained in:
PaaaulZ
@@ -8,12 +8,9 @@ import java.net.*;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import com.oracle.truffle.js.nodes.access.IteratorStepNode;
|
||||
import com.oracle.truffle.js.runtime.builtins.JSON;
|
||||
import org.java_websocket.client.WebSocketClient;
|
||||
|
||||
import org.apache.http.NameValuePair;
|
||||
@@ -61,55 +58,31 @@ public class ScrolllerRipper extends AbstractJSONRipper {
|
||||
|
||||
private JSONObject prepareQuery(String iterator, String gid, String sortByString) throws IOException, URISyntaxException {
|
||||
|
||||
// Prepares the JSONObject we need to pass to the GraphQL query.
|
||||
String QUERY_NOSORT = "query SubredditQuery( $url: String! $filter: SubredditPostFilter $iterator: String ) { getSubreddit(url: $url) { children( limit: 50 iterator: $iterator filter: $filter ) { iterator items { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } } } }";
|
||||
String QUERY_SORT = "subscription SubredditSubscription( $url: String! $sortBy: SubredditSortBy $timespan: SubredditTimespan $iterator: String $limit: Int $filter: SubredditPostFilter ) { fetchSubreddit( url: $url sortBy: $sortBy timespan: $timespan iterator: $iterator limit: $limit filter: $filter ) { __typename ... on Subreddit { __typename url title secondaryTitle description createdAt isNsfw subscribers isComplete itemCount videoCount pictureCount albumCount isFollowing } ... on SubredditPost { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } ... on Iterator { iterator } ... on Error { message } } }";
|
||||
|
||||
if (sortByString.equals("")) {
|
||||
// Sorting not selected
|
||||
String queryString = "query SubredditQuery( $url: String! $filter: SubredditPostFilter $iterator: String ) { getSubreddit(url: $url) { children( limit: 50 iterator: $iterator filter: $filter ) { iterator items { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } } } }";
|
||||
String filterString = convertFilterString(getParameter(this.url,"filter"));
|
||||
String filterString = convertFilterString(getParameter(this.url,"filter"));
|
||||
|
||||
JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid));
|
||||
JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid)).put("sortBy", sortByString.toUpperCase());
|
||||
JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", sortByString.equals("") ? QUERY_NOSORT : QUERY_SORT);
|
||||
|
||||
if (iterator != null) {
|
||||
// Iterator is not present on the first page
|
||||
variablesObject.put("iterator", iterator);
|
||||
}
|
||||
if (!filterString.equals("NOFILTER")) {
|
||||
// We could also pass filter="" but not including it if not present is cleaner
|
||||
variablesObject.put("filter", filterString);
|
||||
}
|
||||
|
||||
JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", queryString);
|
||||
|
||||
return getPosts(finalQueryObject);
|
||||
if (iterator != null) {
|
||||
// Iterator is not present on the first page
|
||||
variablesObject.put("iterator", iterator);
|
||||
}
|
||||
else {
|
||||
|
||||
String queryString = "subscription SubredditSubscription( $url: String! $sortBy: SubredditSortBy $timespan: SubredditTimespan $iterator: String $limit: Int $filter: SubredditPostFilter ) { fetchSubreddit( url: $url sortBy: $sortBy timespan: $timespan iterator: $iterator limit: $limit filter: $filter ) { __typename ... on Subreddit { __typename url title secondaryTitle description createdAt isNsfw subscribers isComplete itemCount videoCount pictureCount albumCount isFollowing } ... on SubredditPost { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } ... on Iterator { iterator } ... on Error { message } } }";
|
||||
String filterString = convertFilterString(getParameter(this.url,"filter"));
|
||||
|
||||
JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid)).put("sortBy", sortByString.toUpperCase());
|
||||
|
||||
if (iterator != null) {
|
||||
// Iterator is not present on the first page
|
||||
variablesObject.put("iterator", iterator);
|
||||
}
|
||||
if (!filterString.equals("NOFILTER")) {
|
||||
// We could also pass filter="" but not including it if not present is cleaner
|
||||
variablesObject.put("filter", filterString);
|
||||
}
|
||||
|
||||
JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", queryString);
|
||||
|
||||
return getPostsSorted(finalQueryObject);
|
||||
if (!filterString.equals("NOFILTER")) {
|
||||
variablesObject.put("filter", filterString);
|
||||
}
|
||||
|
||||
return sortByString.equals("") ? getPosts(finalQueryObject) : getPostsSorted(finalQueryObject);
|
||||
|
||||
}
|
||||
|
||||
|
||||
public String convertFilterString(String filterParameter) {
|
||||
// Converts the ?filter= parameter of the URL to one that can be used in the GraphQL query
|
||||
// I could basically remove the last "s" and call toUpperCase instead of this switch statement but this looks easier to read.
|
||||
switch (filterParameter) {
|
||||
switch (filterParameter.toLowerCase()) {
|
||||
case "pictures":
|
||||
return "PICTURE";
|
||||
case "videos":
|
||||
@@ -134,7 +107,7 @@ public class ScrolllerRipper extends AbstractJSONRipper {
|
||||
// 1) arg.getName() => https://scrolller.com/r/CatsStandingUp?filter
|
||||
// 2) arg.getName() => sort
|
||||
|
||||
if (arg.getName().replace(toReplace,"").equals((parameter))) {
|
||||
if (arg.getName().replace(toReplace,"").toLowerCase().equals((parameter))) {
|
||||
return arg.getValue();
|
||||
}
|
||||
}
|
||||
@@ -198,8 +171,7 @@ public class ScrolllerRipper extends AbstractJSONRipper {
|
||||
@Override
|
||||
public void onMessage(String s) {
|
||||
postsJsonStrings.add(s);
|
||||
if (s.contains("{\"data\":{\"fetchSubreddit\":{\"__typename\":\"Iterator\",\"iterator\":")) {
|
||||
// Iterator is the last field returned, once we received it we can close the connection.
|
||||
if (new JSONObject(s).getJSONObject("data").getJSONObject("fetchSubreddit").has("iterator")) {
|
||||
this.close();
|
||||
}
|
||||
}
|
||||
@@ -230,6 +202,11 @@ public class ScrolllerRipper extends AbstractJSONRipper {
|
||||
}
|
||||
finalObject.put("posts", posts);
|
||||
|
||||
if (finalObject.getJSONArray("posts").length() == 1 && !finalObject.getJSONArray("posts").getJSONObject(0).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) {
|
||||
// Only iterator, no posts.
|
||||
return null;
|
||||
}
|
||||
|
||||
return finalObject;
|
||||
|
||||
|
||||
@@ -244,66 +221,38 @@ public class ScrolllerRipper extends AbstractJSONRipper {
|
||||
@Override
|
||||
protected List<String> getURLsFromJSON(JSONObject json) throws JSONException {
|
||||
|
||||
if (json.has("posts")) {
|
||||
// If the JSONObject contains the key "posts" it's our custom JSON made after the WebSocket call.
|
||||
boolean sortRequested = json.has("posts");
|
||||
|
||||
JSONArray itemsList = json.getJSONArray("posts");
|
||||
int bestArea = 0;
|
||||
String bestUrl = "";
|
||||
List<String> list = new ArrayList<>();
|
||||
int bestArea = 0;
|
||||
String bestUrl = "";
|
||||
List<String> list = new ArrayList<>();
|
||||
|
||||
JSONArray itemsList = sortRequested ? json.getJSONArray("posts") : json.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").getJSONArray("items");
|
||||
|
||||
for (Object item : itemsList) {
|
||||
if (((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) {
|
||||
// Is it really a post? It could be the subreddit description or the iterator (first and last item)
|
||||
JSONArray sourcesTMP = ((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").getJSONArray("mediaSources");
|
||||
for (Object sourceTMP : sourcesTMP)
|
||||
{
|
||||
int widthTMP = ((JSONObject) sourceTMP).getInt("width");
|
||||
int heightTMP = ((JSONObject) sourceTMP).getInt("height");
|
||||
int areaTMP = widthTMP * heightTMP;
|
||||
for (Object item : itemsList) {
|
||||
|
||||
if (areaTMP > bestArea) {
|
||||
// Better way to determine best image?
|
||||
bestArea = widthTMP;
|
||||
bestUrl = ((JSONObject) sourceTMP).getString("url");
|
||||
}
|
||||
}
|
||||
list.add(bestUrl);
|
||||
bestUrl = "";
|
||||
bestArea = 0;
|
||||
if (sortRequested && !((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
JSONArray sourcesTMP = sortRequested ? ((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").getJSONArray("mediaSources") : ((JSONObject) item).getJSONArray("mediaSources");
|
||||
for (Object sourceTMP : sourcesTMP)
|
||||
{
|
||||
int widthTMP = ((JSONObject) sourceTMP).getInt("width");
|
||||
int heightTMP = ((JSONObject) sourceTMP).getInt("height");
|
||||
int areaTMP = widthTMP * heightTMP;
|
||||
|
||||
if (areaTMP > bestArea) {
|
||||
bestArea = widthTMP;
|
||||
bestUrl = ((JSONObject) sourceTMP).getString("url");
|
||||
}
|
||||
}
|
||||
return list;
|
||||
|
||||
} else {
|
||||
JSONArray itemsList = json.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").getJSONArray("items");
|
||||
int bestArea = 0;
|
||||
String bestUrl = "";
|
||||
List<String> list = new ArrayList<>();
|
||||
|
||||
|
||||
for (Object item : itemsList) {
|
||||
JSONArray sourcesTMP = ((JSONObject) item).getJSONArray("mediaSources");
|
||||
for (Object sourceTMP : sourcesTMP)
|
||||
{
|
||||
int widthTMP = ((JSONObject) sourceTMP).getInt("width");
|
||||
int heightTMP = ((JSONObject) sourceTMP).getInt("height");
|
||||
int areaTMP = widthTMP * heightTMP;
|
||||
|
||||
if (areaTMP > bestArea) {
|
||||
// Better way to determine best image?
|
||||
bestArea = widthTMP;
|
||||
bestUrl = ((JSONObject) sourceTMP).getString("url");
|
||||
}
|
||||
}
|
||||
list.add(bestUrl);
|
||||
bestUrl = "";
|
||||
bestArea = 0;
|
||||
}
|
||||
return list;
|
||||
list.add(bestUrl);
|
||||
bestUrl = "";
|
||||
bestArea = 0;
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -323,12 +272,14 @@ public class ScrolllerRipper extends AbstractJSONRipper {
|
||||
|
||||
Object iterator = null;
|
||||
if (source.has("iterator")) {
|
||||
// sorted
|
||||
// Sort requested, custom JSON.
|
||||
iterator = source.getJSONObject("iterator").getJSONObject("data").getJSONObject("fetchSubreddit").get("iterator");
|
||||
} else {
|
||||
iterator = source.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").get("iterator");
|
||||
}
|
||||
|
||||
if (!iterator.toString().equals("null")) {
|
||||
// Need to change page.
|
||||
try {
|
||||
return prepareQuery(iterator.toString(), this.getGID(url), getParameter(url,"sort"));
|
||||
} catch (URISyntaxException e) {
|
||||
@@ -339,5 +290,4 @@ public class ScrolllerRipper extends AbstractJSONRipper {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user