From adffa2261da3f039b5b3a17900053e8a737e5bf6 Mon Sep 17 00:00:00 2001 From: MetaPrime Date: Mon, 6 Jan 2025 02:16:54 -0800 Subject: [PATCH] RedgifsRipper.java - add rate limiting --- .../com/rarchives/ripme/ripper/rippers/RedgifsRipper.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java index d61e17bc..3fc6d669 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java @@ -128,6 +128,8 @@ public class RedgifsRipper extends AbstractJSONRipper { @Override public void downloadURL(URL url, int index) { + // redgifs is easy to trigger rate limit, so be a little cautious + sleep(3000); addURLToDownload(url, getPrefix(index)); } @@ -179,6 +181,9 @@ public class RedgifsRipper extends AbstractJSONRipper { @Override public JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException { + // rate limit on getting next page to not look too much like a crawler/bot + sleep(1000); + if (currentPage == maxPages || isSingleton().matches()) { return null; }