From da469aea4fb4237468f6a81ca088cc4b2f2bd5eb Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 11 Jan 2018 16:07:57 -0500 Subject: [PATCH 1/3] Instagram ripper no longer 403s on certain images --- .../com/rarchives/ripme/ripper/rippers/InstagramRipper.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index ec7825e9..076fcfc6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -161,6 +161,8 @@ public class InstagramRipper extends AbstractHTMLRipper { } private String getOriginalUrl(String imageURL) { + // Without this regex most images will return a 403 error + imageURL = imageURL.replaceAll("vp/[a-zA-Z0-9]*/", ""); imageURL = imageURL.replaceAll("scontent.cdninstagram.com/hphotos-", "igcdn-photos-d-a.akamaihd.net/hphotos-ak-"); // TODO replace this with a single regex imageURL = imageURL.replaceAll("p150x150/", ""); @@ -177,6 +179,7 @@ public class InstagramRipper extends AbstractHTMLRipper { imageURL = imageURL.replaceAll("s720x720/", ""); imageURL = imageURL.replaceAll("s1080x1080/", ""); imageURL = imageURL.replaceAll("s2048x2048/", ""); + // Instagram returns cropped images to unauthenticated applications to maintain legacy support. // To retrieve the uncropped image, remove this segment from the URL. @@ -232,7 +235,7 @@ public class InstagramRipper extends AbstractHTMLRipper { if (imageURLs.size() == 0) { // We add this one item to the array because either wise // the ripper will error out because we returned an empty array - imageURLs.add(data.getString("thumbnail_src")); + imageURLs.add(getOriginalUrl(data.getString("thumbnail_src"))); } addURLToDownload(new URL(getOriginalUrl(data.getString("thumbnail_src"))), image_date); } else { From f1e6d33a1ecac11aee9e8f5e85acdd465cc4f768 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Thu, 11 Jan 2018 16:31:15 -0500 Subject: [PATCH 2/3] 1.7.12: Instagram ripper no longer 403s on certain images --- pom.xml | 2 +- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 17c923d4..fe587fb9 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.7.11 + 1.7.12 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index b2520900..55b96a97 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "latestVersion": "1.7.11", + "latestVersion": "1.7.12", "changeList": [ + "1.7.12: Instagram ripper no longer 403s on certain images", "1.7.11: Added gwarchives support to the cheveretoRipper; Gfycat Tests & Fix for bad reddit submissions; instagram ripper can now be made to skip videos", "1.7.10: Added basic pornpics.com ripper; Fixed hentai.cafe regex", "1.7.9: FuraffinityRipper can now rip non-public albums; Added 2 new api keys, ripper can now download raw images from tumblr; Erome ripper now matchs links without the www; Tumblr ripper now tells the user if it hits the rate limit", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index bf4508bd..15654f5a 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -21,7 +21,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.11"; + private static final String DEFAULT_VERSION = "1.7.12"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static final String mainFileName = "ripme.jar"; From 375f9d8943983bd672d553ffdc93b6e37b688780 Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Wed, 17 Jan 2018 21:05:02 -0500 Subject: [PATCH 3/3] Tumblr API Key Choosing Fix Fixes API choosing. Creates Random objects, generates a number, then feeds that number to API_KEY. This is as opposed to the previous way where it created the Random object, generated a number, then fed it all in a single line. --- .../ripme/ripper/rippers/TumblrRipper.java | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java index 7a8dff75..dc57c48f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java @@ -34,23 +34,30 @@ public class TumblrRipper extends AlbumRipper { private ALBUM_TYPE albumType; private String subdomain, tagName, postNumber; - private static String TUMBLR_AUTH_CONFIG_KEY = "tumblr.auth"; + private static final String TUMBLR_AUTH_CONFIG_KEY = "tumblr.auth"; private static boolean useDefaultApiKey = false; // fall-back for bad user-specified key - private static final List apiKeys = Arrays.asList("JFNLu3CbINQjRdUvZibXW9VpSEVYYtiPJ86o8YmvgLZIoKyuNX", - "FQrwZMCxVnzonv90rgNUJcAk4FpnoS0mYuSuGYqIpM2cFgp9L4", - "qpdkY6nMknksfvYAhf2xIHp0iNRLkMlcWShxqzXyFJRxIsZ1Zz"); - private static final String API_KEY = apiKeys.get(new Random().nextInt(apiKeys.size())); + private static final List APIKEYS = Arrays.asList("JFNLu3CbINQjRdUvZibXW9VpSEVYYtiPJ86o8YmvgLZIoKyuNX", + "FQrwZMCxVnzonv90rgNUJcAk4FpnoS0mYuSuGYqIpM2cFgp9L4", + "qpdkY6nMknksfvYAhf2xIHp0iNRLkMlcWShxqzXyFJRxIsZ1Zz"); + private static int genNum = new Random().nextInt(APIKEYS.size()); + private static final String API_KEY = APIKEYS.get(genNum); // Select random API key from APIKEYS - - private static String getApiKey() { + /** + * Gets the API key. + * Chooses between default/included keys & user specified ones (from the config file). + * @return Tumblr API key + */ + public static String getApiKey() { if (useDefaultApiKey || Utils.getConfigString(TUMBLR_AUTH_CONFIG_KEY, "JFNLu3CbINQjRdUvZibXW9VpSEVYYtiPJ86o8YmvgLZIoKyuNX").equals("JFNLu3CbINQjRdUvZibXW9VpSEVYYtiPJ86o8YmvgLZIoKyuNX")) { logger.info("Using api key: " + API_KEY); return API_KEY; } else { - logger.info("Using user tumblr.auth api key"); - return Utils.getConfigString(TUMBLR_AUTH_CONFIG_KEY, "JFNLu3CbINQjRdUvZibXW9VpSEVYYtiPJ86o8YmvgLZIoKyuNX"); + String userDefinedAPIKey = Utils.getConfigString(TUMBLR_AUTH_CONFIG_KEY, "JFNLu3CbINQjRdUvZibXW9VpSEVYYtiPJ86o8YmvgLZIoKyuNX"); + logger.info("Using user tumblr.auth api key: " + userDefinedAPIKey); + return userDefinedAPIKey; } + } public TumblrRipper(URL url) throws IOException { @@ -64,7 +71,13 @@ public class TumblrRipper extends AlbumRipper { public boolean canRip(URL url) { return url.getHost().endsWith(DOMAIN); } - + + /** + * Sanitizes URL. + * @param url URL to be sanitized. + * @return Sanitized URL + * @throws MalformedURLException + */ @Override public URL sanitizeURL(URL url) throws MalformedURLException { String u = url.toExternalForm();