mirror/ripme
1
0
Fork 0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-08-15 10:24:04 +02:00
Code Issues Releases Wiki Activity

Fix rgif ripper

Browse Source
This commit is contained in:
Tush-r
2024-08-29 17:21:01 +05:30
committed by soloturn
parent aa00c6f612
commit 488849c253
1 changed files with 74 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java
View File

@@ -1,10 +1,8 @@
package com.rarchives.ripme.ripper.rippers; package com.rarchives.ripme.ripper.rippers;
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Http;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONObject; import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.select.Elements; import org.jsoup.select.Elements;
@@ -19,14 +17,26 @@ import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
public class RedgifsRipper extends AbstractHTMLRipper { import org.apache.http.client.utils.URIBuilder;
import com.rarchives.ripme.ripper.AbstractJSONRipper;
public class RedgifsRipper extends AbstractJSONRipper {
private static final String HOST = "redgifs.com"; private static final String HOST = "redgifs.com";
private static final String HOST_2 = "gifdeliverynetwork.com"; private static final String HOST_2 = "gifdeliverynetwork.com";
private static final String GIFS_DETAIL_ENDPOINT = "https://api.redgifs.com/v2/gifs/%s";
private static final String USERS_SEARCH_ENDPOINT = "https://api.redgifs.com/v2/users/%s/search";
private static final String TEMPORARY_AUTH_ENDPOINT = "https://api.redgifs.com/v2/auth/temporary";
String username = ""; String username = "";
String authToken = "";
// TODO remove
String cursor = ""; String cursor = "";
String count = "100"; int count = 40;
int currentPage = 1;
int maxPages = 1;
// TODO remove with search
String searchText = ""; String searchText = "";
int searchCount = 150; int searchCount = 150;
int searchStart = 0; int searchStart = 0;
@@ -73,18 +83,29 @@ public class RedgifsRipper extends AbstractHTMLRipper {
} }
@Override @Override
public Document getFirstPage() throws IOException { public JSONObject getFirstPage() throws IOException {
try { try {
if (!isProfile().matches() && !isSearch().matches()) { if (authToken == null || authToken.equals("")){
return Http.url(url).get(); fetchAuthToken();
}
if (isSingleton().matches()) {
maxPages = 1;
String gifDetailsURL = String.format(GIFS_DETAIL_ENDPOINT, getGID(url));
return Http.url(gifDetailsURL).header("Authorization", "Bearer " + authToken).getJSON();
} else if (isSearch().matches()) { } else if (isSearch().matches()) {
searchText = getGID(url).replace("-", " "); // TODO fix search
return Http.url( // TODO remove
new URI("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart * searchCount).toURL()).ignoreContentType().get(); throw new IOException("TODO remove");
} else { } else {
username = getGID(url); username = getGID(url);
return Http.url(new URI("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count).toURL()) var uri = new URIBuilder(String.format(USERS_SEARCH_ENDPOINT, username));
.ignoreContentType().get(); uri.addParameter("order", "new");
uri.addParameter("count", Integer.toString(count));
uri.addParameter("page", Integer.toString(currentPage));
var json = Http.url(uri.build().toURL()).header("Authorization", "Bearer " + authToken).getJSON();
maxPages = json.getInt("pages");
return json;
} }
} catch (URISyntaxException e) { } catch (URISyntaxException e) {
throw new IOException(e); throw new IOException(e);
@@ -118,6 +139,7 @@ public class RedgifsRipper extends AbstractHTMLRipper {
+ " Got: " + url); + " Got: " + url);
} }
// TODO remove
private String stripHTMLTags(String t) { private String stripHTMLTags(String t) {
t = t.replaceAll("<html>\n" + t = t.replaceAll("<html>\n" +
" <head></head>\n" + " <head></head>\n" +
@@ -130,42 +152,47 @@ public class RedgifsRipper extends AbstractHTMLRipper {
} }
@Override @Override
public Document getNextPage(Document doc) throws IOException, URISyntaxException { public JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException {
if (isSearch().matches()) { if (currentPage == maxPages || isSingleton().matches()){
Document d = Http.url(
new URI("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText
+ "&count=" + searchCount + "&start=" + searchCount*++searchStart).toURL())
.ignoreContentType().get();
return (hasURLs(d).isEmpty()) ? null : d;
} else {
if (cursor.equals("") || cursor.equals("null")) {
return null; return null;
} else {
Document d = Http.url(new URI("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor).toURL()).ignoreContentType().get();
return (hasURLs(d).isEmpty()) ? null : d;
} }
currentPage++;
if (isSearch().matches()) {
// TODO search
// TODO remove
throw new IOException("// TODO remove");
} else if (isProfile().matches()) {
var uri = new URIBuilder(String.format(USERS_SEARCH_ENDPOINT, getGID(url)));
uri.addParameter("order", "new");
uri.addParameter("count", Integer.toString(count));
uri.addParameter("page", Integer.toString(currentPage));
var json = Http.url(uri.build().toURL()).header("Authorization", "Bearer " + authToken).getJSON();
// Handle rare maxPages change during a rip
maxPages = json.getInt("pages");
return json;
} else {
return null;
} }
} }
@Override @Override
public List<String> getURLsFromPage(Document doc) { public List<String> getURLsFromJSON(JSONObject json) {
List<String> result = new ArrayList<>(); List<String> result = new ArrayList<>();
if (isProfile().matches() || isSearch().matches()) { if (isProfile().matches() || isSearch().matches()) {
result = hasURLs(doc); // TODO check json keys for search
var gifs = json.getJSONArray("gifs");
for (var gif : gifs){
var hdURL = ((JSONObject)gif).getJSONObject("urls").getString("hd");
result.add(hdURL);
}
} else { } else {
Elements videos = doc.select("script"); String hdURL = json.getJSONObject("gif").getJSONObject("urls").getString("hd");
for (Element el : videos) { result.add(hdURL);
String json = el.html();
if (json.startsWith("{")) {
JSONObject page = new JSONObject(json);
result.add(page.getJSONObject("video").getString("contentUrl")
.replace("-mobile", ""));
}
}
} }
return result; return result;
} }
// TODO delete
/** /**
* Helper method for retrieving URLs. * Helper method for retrieving URLs.
* @param doc Document of the URL page to look through * @param doc Document of the URL page to look through
@@ -182,6 +209,7 @@ public class RedgifsRipper extends AbstractHTMLRipper {
return result; return result;
} }
// TODO delete
/** /**
* Helper method for retrieving video URLs. * Helper method for retrieving video URLs.
* @param url URL to gfycat page * @param url URL to gfycat page
@@ -206,4 +234,15 @@ public class RedgifsRipper extends AbstractHTMLRipper {
} }
throw new IOException(); throw new IOException();
} }
/**
* Fetch a temorary auth token for the rip
* @throws IOException
*/
private void fetchAuthToken() throws IOException{
var json = Http.url(TEMPORARY_AUTH_ENDPOINT).getJSON();
var token = json.getString("token");
authToken = token;
}
} }