mirror/ripme
1
0
Fork 0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-08-12 08:54:11 +02:00
Code Issues Releases Wiki Activity

Merge pull request #1404 from Isaaku/issues/twitter_fix

Browse Source 
Fix twitter video dimension
This commit is contained in:
cyian-1756
2019-08-30 14:52:51 -05:00
committed by GitHub
parent 77397519b0 8101754af5
commit 0fb8e83652
1 changed files with 50 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
View File

@@ -22,8 +22,7 @@ public class TwitterRipper extends AlbumRipper {
int downloadUrls = 1; int downloadUrls = 1;
private static final String DOMAIN = "twitter.com", private static final String DOMAIN = "twitter.com", HOST = "twitter";
HOST = "twitter";
private static final int MAX_REQUESTS = Utils.getConfigInteger("twitter.max_requests", 10); private static final int MAX_REQUESTS = Utils.getConfigInteger("twitter.max_requests", 10);
private static final boolean RIP_RETWEETS = Utils.getConfigBoolean("twitter.rip_retweets", true); private static final boolean RIP_RETWEETS = Utils.getConfigBoolean("twitter.rip_retweets", true);
@@ -34,8 +33,7 @@ public class TwitterRipper extends AlbumRipper {
private String accessToken; private String accessToken;
private enum ALBUM_TYPE { private enum ALBUM_TYPE {
ACCOUNT, ACCOUNT, SEARCH
SEARCH
} }
private ALBUM_TYPE albumType; private ALBUM_TYPE albumType;
@@ -75,13 +73,10 @@ public class TwitterRipper extends AlbumRipper {
} }
private void getAccessToken() throws IOException { private void getAccessToken() throws IOException {
Document doc = Http.url("https://api.twitter.com/oauth2/token") Document doc = Http.url("https://api.twitter.com/oauth2/token").ignoreContentType()
.ignoreContentType()
.header("Authorization", "Basic " + authKey) .header("Authorization", "Basic " + authKey)
.header("Content-Type", "application/x-www-form-urlencoded;charset=UTF-8") .header("Content-Type", "application/x-www-form-urlencoded;charset=UTF-8")
.header("User-agent", "ripe and zipe") .header("User-agent", "ripe and zipe").data("grant_type", "client_credentials").post();
.data("grant_type", "client_credentials")
.post();
String body = doc.body().html().replaceAll(""", "\""); String body = doc.body().html().replaceAll(""", "\"");
try { try {
JSONObject json = new JSONObject(body); JSONObject json = new JSONObject(body);
@@ -94,17 +89,13 @@ public class TwitterRipper extends AlbumRipper {
private void checkRateLimits(String resource, String api) throws IOException { private void checkRateLimits(String resource, String api) throws IOException {
Document doc = Http.url("https://api.twitter.com/1.1/application/rate_limit_status.json?resources=" + resource) Document doc = Http.url("https://api.twitter.com/1.1/application/rate_limit_status.json?resources=" + resource)
.ignoreContentType() .ignoreContentType().header("Authorization", "Bearer " + accessToken)
.header("Authorization", "Bearer " + accessToken)
.header("Content-Type", "application/x-www-form-urlencoded;charset=UTF-8") .header("Content-Type", "application/x-www-form-urlencoded;charset=UTF-8")
.header("User-agent", "ripe and zipe") .header("User-agent", "ripe and zipe").get();
.get();
String body = doc.body().html().replaceAll(""", "\""); String body = doc.body().html().replaceAll(""", "\"");
try { try {
JSONObject json = new JSONObject(body); JSONObject json = new JSONObject(body);
JSONObject stats = json.getJSONObject("resources") JSONObject stats = json.getJSONObject("resources").getJSONObject(resource).getJSONObject(api);
.getJSONObject(resource)
.getJSONObject(api);
int remaining = stats.getInt("remaining"); int remaining = stats.getInt("remaining");
LOGGER.info(" Twitter " + resource + " calls remaining: " + remaining); LOGGER.info(" Twitter " + resource + " calls remaining: " + remaining);
if (remaining < 20) { if (remaining < 20) {
@@ -120,23 +111,17 @@ public class TwitterRipper extends AlbumRipper {
private String getApiURL(Long maxID) { private String getApiURL(Long maxID) {
StringBuilder req = new StringBuilder(); StringBuilder req = new StringBuilder();
switch (albumType) { switch (albumType) {
case ACCOUNT: case ACCOUNT:
req.append("https://api.twitter.com/1.1/statuses/user_timeline.json") req.append("https://api.twitter.com/1.1/statuses/user_timeline.json")
.append("?screen_name=" + this.accountName) .append("?screen_name=" + this.accountName).append("&include_entities=true")
.append("&include_entities=true") .append("&exclude_replies=true").append("&trim_user=true").append("&count=" + 200)
.append("&exclude_replies=true") .append("&tweet_mode=extended");
.append("&trim_user=true") break;
.append("&count=" + 200) case SEARCH:
.append("&tweet_mode=extended"); req.append("https://api.twitter.com/1.1/search/tweets.json").append("?q=" + this.searchText)
break; .append("&include_entities=true").append("&result_type=recent").append("&count=100")
case SEARCH: .append("&tweet_mode=extended");
req.append("https://api.twitter.com/1.1/search/tweets.json") break;
.append("?q=" + this.searchText)
.append("&include_entities=true")
.append("&result_type=recent")
.append("&count=100")
.append("&tweet_mode=extended");
break;
} }
if (maxID > 0) { if (maxID > 0) {
req.append("&max_id=" + Long.toString(maxID)); req.append("&max_id=" + Long.toString(maxID));
@@ -147,12 +132,9 @@ public class TwitterRipper extends AlbumRipper {
private List<JSONObject> getTweets(String url) throws IOException { private List<JSONObject> getTweets(String url) throws IOException {
List<JSONObject> tweets = new ArrayList<>(); List<JSONObject> tweets = new ArrayList<>();
LOGGER.info(" Retrieving " + url); LOGGER.info(" Retrieving " + url);
Document doc = Http.url(url) Document doc = Http.url(url).ignoreContentType().header("Authorization", "Bearer " + accessToken)
.ignoreContentType()
.header("Authorization", "Bearer " + accessToken)
.header("Content-Type", "application/x-www-form-urlencoded;charset=UTF-8") .header("Content-Type", "application/x-www-form-urlencoded;charset=UTF-8")
.header("User-agent", "ripe and zipe") .header("User-agent", "ripe and zipe").get();
.get();
String body = doc.body().html().replaceAll("&quot;", "\""); String body = doc.body().html().replaceAll("&quot;", "\"");
Object jsonObj = new JSONTokener(body).nextValue(); Object jsonObj = new JSONTokener(body).nextValue();
JSONArray statuses; JSONArray statuses;
@@ -178,7 +160,7 @@ public class TwitterRipper extends AlbumRipper {
LOGGER.error("XXX Tweet doesn't have entitites"); LOGGER.error("XXX Tweet doesn't have entitites");
return 0; return 0;
} }
if (!RIP_RETWEETS && tweet.has("retweeted_status")) { if (!RIP_RETWEETS && tweet.has("retweeted_status")) {
LOGGER.info("Skipping a retweet as twitter.rip_retweet is set to false."); LOGGER.info("Skipping a retweet as twitter.rip_retweet is set to false.");
return 0; return 0;
@@ -194,19 +176,22 @@ public class TwitterRipper extends AlbumRipper {
for (int i = 0; i < medias.length(); i++) { for (int i = 0; i < medias.length(); i++) {
media = (JSONObject) medias.get(i); media = (JSONObject) medias.get(i);
url = media.getString("media_url"); url = media.getString("media_url");
if (media.getString("type").equals("video")) { if (media.getString("type").equals("video") || media.getString("type").equals("animated_gif")) {
JSONArray variants = media.getJSONObject("video_info").getJSONArray("variants"); JSONArray variants = media.getJSONObject("video_info").getJSONArray("variants");
int largestBitrate = 0; int largestBitrate = 0;
String urlToDownload = null; String urlToDownload = null;
// Loop over all the video options and find the biggest video // Loop over all the video options and find the biggest video
for (int j = 0; j < medias.length(); j++) { for (int j = 0; j < variants.length(); j++) {
JSONObject variant = (JSONObject) variants.get(i); JSONObject variant = (JSONObject) variants.get(j);
LOGGER.info(variant); LOGGER.info(variant);
// If the video doesn't have a bitrate it's a m3u8 file we can't download // If the video doesn't have a bitrate it's a m3u8 file we can't download
if (variant.has("bitrate")) { if (variant.has("bitrate")) {
if (variant.getInt("bitrate") > largestBitrate) { if (variant.getInt("bitrate") > largestBitrate) {
largestBitrate = variant.getInt("bitrate"); largestBitrate = variant.getInt("bitrate");
urlToDownload = variant.getString("url"); urlToDownload = variant.getString("url");
} else if (media.getString("type").equals("animated_gif")) {
// If the type if animated_gif the bitrate doesn't matter
urlToDownload = variant.getString("url");
} }
} }
} }
@@ -230,12 +215,11 @@ public class TwitterRipper extends AlbumRipper {
} }
} }
return parsedCount; return parsedCount;
} }
public String getPrefix(int index) { public String getPrefix(int index) {
return String.format("%03d_", index); return Utils.getConfigBoolean("download.save_order", true) ? String.format("%03d_", index) : "";
} }
@Override @Override
@@ -243,12 +227,12 @@ public class TwitterRipper extends AlbumRipper {
getAccessToken(); getAccessToken();
switch (albumType) { switch (albumType) {
case ACCOUNT: case ACCOUNT:
checkRateLimits("statuses", "/statuses/user_timeline"); checkRateLimits("statuses", "/statuses/user_timeline");
break; break;
case SEARCH: case SEARCH:
checkRateLimits("search", "/search/tweets"); checkRateLimits("search", "/search/tweets");
break; break;
} }
Long lastMaxID = 0L; Long lastMaxID = 0L;
@@ -260,9 +244,7 @@ public class TwitterRipper extends AlbumRipper {
break; break;
} }
LOGGER.debug("Twitter response #" + (i + 1) + " Tweets:\n" + tweets); LOGGER.debug("Twitter response #" + (i + 1) + " Tweets:\n" + tweets);
if (tweets.size() == 1 && if (tweets.size() == 1 && lastMaxID.equals(tweets.get(0).getString("id_str"))) {
lastMaxID.equals(tweets.get(0).getString("id_str"))
) {
LOGGER.info(" No more tweet found."); LOGGER.info(" No more tweet found.");
break; break;
} }
@@ -299,26 +281,22 @@ public class TwitterRipper extends AlbumRipper {
@Override @Override
public String getGID(URL url) throws MalformedURLException { public String getGID(URL url) throws MalformedURLException {
switch (albumType) { switch (albumType) {
case ACCOUNT: case ACCOUNT:
return "account_" + accountName; return "account_" + accountName;
case SEARCH: case SEARCH:
StringBuilder gid = new StringBuilder(); StringBuilder gid = new StringBuilder();
for (int i = 0; i < searchText.length(); i++) { for (int i = 0; i < searchText.length(); i++) {
char c = searchText.charAt(i); char c = searchText.charAt(i);
// Ignore URL-encoded chars // Ignore URL-encoded chars
if (c == '%') { if (c == '%') {
gid.append('_'); gid.append('_');
i += 2; i += 2;
// Ignore non-alphanumeric chars // Ignore non-alphanumeric chars
} else if ( } else if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) {
(c >= 'a' && c <= 'z') gid.append(c);
|| (c >= 'A' && c <= 'Z')
|| (c >= '0' && c <= '9')
) {
gid.append(c);
}
} }
return "search_" + gid.toString(); }
return "search_" + gid.toString();
} }
throw new MalformedURLException("Could not decide type of URL (search/account): " + url); throw new MalformedURLException("Could not decide type of URL (search/account): " + url);
} }