diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 6380a1c6..7f3509b1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -342,6 +342,9 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); return false; } + if (shouldIgnoreURL(url)) { + return false; + } if (Utils.getConfigBoolean("urls_only.save", false)) { // Output URL to file Path urlFile = Paths.get(this.workingDir + "/urls.txt"); diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 523a8ab8..31f94cde 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -160,6 +160,9 @@ public abstract class AbstractJSONRipper extends AbstractRipper { LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); return false; } + if (shouldIgnoreURL(url)) { + return false; + } if (Utils.getConfigBoolean("urls_only.save", false)) { // Output URL to file Path urlFile = Paths.get(this.workingDir + "/urls.txt"); diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 1a25af18..fa58b5c1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -699,4 +699,18 @@ public abstract class AbstractRipper protected boolean useByteProgessBar() { return false;} // If true ripme will try to resume a broken download for this ripper protected boolean tryResumeDownload() { return false;} + + protected boolean shouldIgnoreURL(URL url) { + final String[] ignoredExtensions = Utils.getConfigStringArray("download.ignore_extensions"); + if (ignoredExtensions == null || ignoredExtensions.length == 0) return false; // nothing ignored + String[] pathElements = url.getPath().split("\\."); + if (pathElements.length == 0) return false; // no extension, can't filter + String extension = pathElements[pathElements.length - 1]; + for (String ignoredExtension : ignoredExtensions) { + if (ignoredExtension.equalsIgnoreCase(extension)) { + return true; + } + } + return false; + } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 96c352f5..94d506e3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -70,6 +70,9 @@ public abstract class AlbumRipper extends AbstractRipper { LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); return false; } + if (shouldIgnoreURL(url)) { + return false; + } if (Utils.getConfigBoolean("urls_only.save", false)) { // Output URL to file Path urlFile = Paths.get(this.workingDir + "/urls.txt"); diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index cd4c95b3..391ce2f4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -68,6 +68,9 @@ public abstract class VideoRipper extends AbstractRipper { this.url = url; return true; } + if (shouldIgnoreURL(url)) { + return false; + } threadPool.addThread(new DownloadVideoThread(url, saveAs, this)); } return true; diff --git a/src/main/resources/rip.properties b/src/main/resources/rip.properties index cac0c1f1..35d6c123 100644 --- a/src/main/resources/rip.properties +++ b/src/main/resources/rip.properties @@ -17,6 +17,9 @@ page.timeout = 5000 # Maximum size of downloaded files in bytes (required) download.max_size = 104857600 +# Any URLs ending with one of these comma-separated values will be skipped +#download.ignore_extensions = mp4,gif,m4v,webm,html + # Don't retry on 404 errors error.skip404 = true