diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 3e3fdb18..81ef87cd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -6,10 +6,12 @@ import java.io.FileWriter; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; + import org.jsoup.nodes.Document; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; @@ -22,9 +24,9 @@ import com.rarchives.ripme.ui.RipStatusMessage; */ public abstract class AbstractHTMLRipper extends AbstractRipper { - private Map itemsPending = Collections.synchronizedMap(new HashMap()); - private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); - private Map itemsErrored = Collections.synchronizedMap(new HashMap()); + private final Map itemsPending = Collections.synchronizedMap(new HashMap<>()); + private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); + private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); protected AbstractHTMLRipper(URL url) throws IOException { super(url); @@ -104,7 +106,15 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { LOGGER.debug("Adding items from " + this.url + " to queue"); } + List doclocation = new ArrayList<>(); while (doc != null) { + + // catch if we saw a doc location already, save the ones seen in a list + if (doclocation.contains(doc.location())) { + break; + } + doclocation.add(doc.location()); + if (alreadyDownloadedUrls >= Utils.getConfigInteger("history.end_rip_after_already_seen", 1000000000) && !isThisATest()) { sendUpdate(STATUS.DOWNLOAD_COMPLETE_HISTORY, "Already seen the last " + alreadyDownloadedUrls + " images ending rip"); break; @@ -195,7 +205,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { */ private String fileNameFromURL(URL url) { String saveAs = url.toExternalForm(); - if (saveAs.substring(saveAs.length() - 1) == "/") { saveAs = saveAs.substring(0,saveAs.length() - 1) ;} + if (saveAs.substring(saveAs.length() - 1).equals("/")) { saveAs = saveAs.substring(0,saveAs.length() - 1) ;} saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1); if (saveAs.indexOf('?') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('?')); } if (saveAs.indexOf('#') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('#')); } @@ -281,16 +291,16 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { } @Override - /** - * Returns total amount of files attempted. + /* + Returns total amount of files attempted. */ public int getCount() { return itemsCompleted.size() + itemsErrored.size(); } @Override - /** - * Queues multiple URLs of single images to download from a single Album URL + /* + Queues multiple URLs of single images to download from a single Album URL */ public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { // Only download one file if this is a test. @@ -352,8 +362,8 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { } @Override - /** - * Cleans up & tells user about successful download + /* + Cleans up & tells user about successful download */ public void downloadCompleted(URL url, File saveAs) { if (observer == null) { @@ -388,9 +398,9 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { } @Override - /** - * Tells user that a single file in the album they wish to download has - * already been downloaded in the past. + /* + Tells user that a single file in the album they wish to download has + already been downloaded in the past. */ public void downloadExists(URL url, File file) { if (observer == null) { @@ -466,13 +476,12 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { */ @Override public String getStatusText() { - StringBuilder sb = new StringBuilder(); - sb.append(getCompletionPercentage()) - .append("% ") - .append("- Pending: " ).append(itemsPending.size()) - .append(", Completed: ").append(itemsCompleted.size()) - .append(", Errored: " ).append(itemsErrored.size()); - return sb.toString(); + String sb = getCompletionPercentage() + + "% " + + "- Pending: " + itemsPending.size() + + ", Completed: " + itemsCompleted.size() + + ", Errored: " + itemsErrored.size(); + return sb; }