From 62c9be3b4035e71cdefb98b3fceda68bded7f112 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 22 Oct 2019 12:52:23 -0400 Subject: [PATCH 1/2] Imgur ripper now use api.imgur.com when ripping albums to bypass age gate --- .../ripme/ripper/rippers/ImgurRipper.java | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index d3f94456..f3c754fd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -15,6 +15,7 @@ import org.json.JSONObject; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.jsoup.safety.Whitelist; import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AlbumRipper; @@ -272,20 +273,17 @@ public class ImgurRipper extends AlbumRipper { strUrl += "/all"; } LOGGER.info(" Retrieving " + strUrl); - Document doc = getDocument(strUrl); + Document doc = getAlbumData("https://api.imgur.com/3/album/" + strUrl.split("/a/")[1]); // Try to use embedded JSON to retrieve images - Matcher m = getEmbeddedJsonMatcher(doc); - if (m.matches()) { + LOGGER.info(Jsoup.clean(doc.body().toString(), Whitelist.none())); + try { - JSONObject json = new JSONObject(m.group(1)); - JSONArray jsonImages = json.getJSONObject("image") - .getJSONObject("album_images") - .getJSONArray("images"); + JSONObject json = new JSONObject(Jsoup.clean(doc.body().toString(), Whitelist.none())); + JSONArray jsonImages = json.getJSONObject("data").getJSONArray("images"); return createImgurAlbumFromJsonArray(url, jsonImages); } catch (JSONException e) { LOGGER.debug("Error while parsing JSON at " + url + ", continuing", e); } - } // TODO If album is empty, use this to check for cached images: // http://i.rarchives.com/search.cgi?cache=http://imgur.com/a/albumID @@ -332,8 +330,8 @@ public class ImgurRipper extends AlbumRipper { ImgurAlbum imgurAlbum = new ImgurAlbum(url); int imagesLength = jsonImages.length(); for (int i = 0; i < imagesLength; i++) { - JSONObject jsonImage = jsonImages.getJSONObject(i); - imgurAlbum.addImage(createImgurImageFromJson(jsonImage)); + JSONObject ob = jsonImages.getJSONObject(i); + imgurAlbum.addImage(new ImgurImage( new URL(ob.getString("link")))); } return imgurAlbum; } @@ -361,6 +359,17 @@ public class ImgurRipper extends AlbumRipper { .get(); } + private static Document getAlbumData(String strUrl) throws IOException { + return Jsoup.connect(strUrl) + .userAgent(USER_AGENT) + .timeout(10 * 1000) + .maxBodySize(0) + .header("Authorization", "Client-ID 546c25a59c58ad7") + .ignoreContentType(true) + .get(); + } + + /** * Rips all albums in an imgur user's account. * @param url From c589f071506907c386cdbafbd21653f1b9133303 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 22 Oct 2019 12:57:40 -0400 Subject: [PATCH 2/2] Agged the imgur.client_id config option so users can provide their own imgur client-id --- .../java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index f3c754fd..93cb809e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -364,7 +364,7 @@ public class ImgurRipper extends AlbumRipper { .userAgent(USER_AGENT) .timeout(10 * 1000) .maxBodySize(0) - .header("Authorization", "Client-ID 546c25a59c58ad7") + .header("Authorization", "Client-ID " + Utils.getConfigString("imgur.client_id", "546c25a59c58ad7")) .ignoreContentType(true) .get(); }