From 6e86e94bd77d64292f48fafa3ebb1debf1b14bfb Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 13 Nov 2018 00:06:55 -0500 Subject: [PATCH] Started on fixing reddit redownloading issue --- .../com/rarchives/ripme/ripper/AbstractRipper.java | 3 ++- .../java/com/rarchives/ripme/ripper/AlbumRipper.java | 11 ++++++++++- .../rarchives/ripme/ripper/rippers/RedditRipper.java | 3 ++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 95643b4c..3be64665 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -131,7 +131,7 @@ public abstract class AbstractRipper * Returns true if previously downloaded. * Returns false if not yet downloaded. */ - private boolean hasDownloadedURL(String url) { + protected boolean hasDownloadedURL(String url) { File file = new File(URLHistoryFile); url = normalizeUrl(url); @@ -280,6 +280,7 @@ public abstract class AbstractRipper saveFileAs.getParentFile().mkdirs(); } if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) { + LOGGER.info("Writing " + url.toExternalForm() + " to file"); try { writeDownloadedURL(url.toExternalForm() + "\n"); } catch (IOException e) { diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 97943b33..fa5f2ccc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -51,7 +51,16 @@ public abstract class AlbumRipper extends AbstractRipper { * Queues multiple URLs of single images to download from a single Album URL */ public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. + // Don't re-add the url if it was downloaded in a previous rip + if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) { + if (hasDownloadedURL(url.toExternalForm())) { + sendUpdate(STATUS.DOWNLOAD_WARN, "Already downloaded " + url.toExternalForm()); + alreadyDownloadedUrls += 1; + return false; + } + } + + // Only download one file if this is a test. if (super.isThisATest() && (itemsPending.size() > 0 || itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { stop(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 172250e3..5d702fc7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -8,6 +8,7 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.ui.RipStatusMessage; import org.json.JSONArray; import org.json.JSONObject; @@ -208,7 +209,6 @@ public class RedditRipper extends AlbumRipper { largestHeight = Integer.parseInt(height); baseURL = doc.select("MPD > Period > AdaptationSet > Representation[height=" + height + "]").select("BaseURL").text(); } - LOGGER.info("H " + e.attr("height") + " V " + e.attr("width")); } return new URL(vidURL + "/" + baseURL); } catch (IOException e) { @@ -251,6 +251,7 @@ public class RedditRipper extends AlbumRipper { savePath += id + "-" + url.split("/")[3] + title + ".mp4"; URL urlToDownload = parseRedditVideoMPD(urls.get(0).toExternalForm()); if (urlToDownload != null) { + LOGGER.info("url: " + urlToDownload + " file: " + savePath); addURLToDownload(urlToDownload, new File(savePath)); } }