From cfb0366aca54618368e4abebbc8c951dab04ab63 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Mar 2021 08:03:29 +0200 Subject: [PATCH 1/2] abstractripper avoids endless runs by checking url is seen multiple bug reports were made, like #20, RipMeApp#1854 . --- .../com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 3e3fdb18..db18b2e1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -6,10 +6,12 @@ import java.io.FileWriter; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; + import org.jsoup.nodes.Document; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; @@ -104,7 +106,15 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { LOGGER.debug("Adding items from " + this.url + " to queue"); } + List doclocation = new ArrayList<>(); while (doc != null) { + + // catch if we saw a doc location already, save the ones seen in a list + if (doclocation.contains(doc.location())) { + break; + } + doclocation.add(doc.location()); + if (alreadyDownloadedUrls >= Utils.getConfigInteger("history.end_rip_after_already_seen", 1000000000) && !isThisATest()) { sendUpdate(STATUS.DOWNLOAD_COMPLETE_HISTORY, "Already seen the last " + alreadyDownloadedUrls + " images ending rip"); break; From 9518b61b0f38211167e01606e2e2b05b222cabea Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Mar 2021 10:45:34 +0200 Subject: [PATCH 2/2] java8 syntax update of AbstractRipper --- .../ripme/ripper/AbstractHTMLRipper.java | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index db18b2e1..81ef87cd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -24,9 +24,9 @@ import com.rarchives.ripme.ui.RipStatusMessage; */ public abstract class AbstractHTMLRipper extends AbstractRipper { - private Map itemsPending = Collections.synchronizedMap(new HashMap()); - private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); - private Map itemsErrored = Collections.synchronizedMap(new HashMap()); + private final Map itemsPending = Collections.synchronizedMap(new HashMap<>()); + private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); + private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); protected AbstractHTMLRipper(URL url) throws IOException { super(url); @@ -205,7 +205,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { */ private String fileNameFromURL(URL url) { String saveAs = url.toExternalForm(); - if (saveAs.substring(saveAs.length() - 1) == "/") { saveAs = saveAs.substring(0,saveAs.length() - 1) ;} + if (saveAs.substring(saveAs.length() - 1).equals("/")) { saveAs = saveAs.substring(0,saveAs.length() - 1) ;} saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1); if (saveAs.indexOf('?') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('?')); } if (saveAs.indexOf('#') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('#')); } @@ -291,16 +291,16 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { } @Override - /** - * Returns total amount of files attempted. + /* + Returns total amount of files attempted. */ public int getCount() { return itemsCompleted.size() + itemsErrored.size(); } @Override - /** - * Queues multiple URLs of single images to download from a single Album URL + /* + Queues multiple URLs of single images to download from a single Album URL */ public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { // Only download one file if this is a test. @@ -362,8 +362,8 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { } @Override - /** - * Cleans up & tells user about successful download + /* + Cleans up & tells user about successful download */ public void downloadCompleted(URL url, File saveAs) { if (observer == null) { @@ -398,9 +398,9 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { } @Override - /** - * Tells user that a single file in the album they wish to download has - * already been downloaded in the past. + /* + Tells user that a single file in the album they wish to download has + already been downloaded in the past. */ public void downloadExists(URL url, File file) { if (observer == null) { @@ -476,13 +476,12 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { */ @Override public String getStatusText() { - StringBuilder sb = new StringBuilder(); - sb.append(getCompletionPercentage()) - .append("% ") - .append("- Pending: " ).append(itemsPending.size()) - .append(", Completed: ").append(itemsCompleted.size()) - .append(", Errored: " ).append(itemsErrored.size()); - return sb.toString(); + String sb = getCompletionPercentage() + + "% " + + "- Pending: " + itemsPending.size() + + ", Completed: " + itemsCompleted.size() + + ", Errored: " + itemsErrored.size(); + return sb; }