mirror/ripme
1
0
Fork 0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-08-23 05:53:02 +02:00
Code Issues Releases Wiki Activity

abstractripper avoids endless runs by checking url is seen

Browse Source 
multiple bug reports were made, like #20, RipMeApp#1854 .
This commit is contained in:
soloturn
2021-03-28 08:03:29 +02:00
parent 7f1ae26d9b
commit cfb0366aca
1 changed files with 10 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java
View File

@@ -6,10 +6,12 @@ import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
@@ -104,7 +106,15 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
LOGGER.debug("Adding items from " + this.url + " to queue"); LOGGER.debug("Adding items from " + this.url + " to queue");
} }
List<String> doclocation = new ArrayList<>();
while (doc != null) { while (doc != null) {
// catch if we saw a doc location already, save the ones seen in a list
if (doclocation.contains(doc.location())) {
break;
}
doclocation.add(doc.location());
if (alreadyDownloadedUrls >= Utils.getConfigInteger("history.end_rip_after_already_seen", 1000000000) && !isThisATest()) { if (alreadyDownloadedUrls >= Utils.getConfigInteger("history.end_rip_after_already_seen", 1000000000) && !isThisATest()) {
sendUpdate(STATUS.DOWNLOAD_COMPLETE_HISTORY, "Already seen the last " + alreadyDownloadedUrls + " images ending rip"); sendUpdate(STATUS.DOWNLOAD_COMPLETE_HISTORY, "Already seen the last " + alreadyDownloadedUrls + " images ending rip");
break; break;