mirror/ripme
1
0
Fork 0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-08-16 19:04:05 +02:00
Code Issues Releases Wiki Activity

Skip specified URL file extensions

Browse Source
This commit is contained in:
jpoulton
2023-11-04 15:45:39 +00:00
committed by soloturn
parent ce0e60c501
commit ea8e4cdfea
6 changed files with 29 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java
View File

@@ -342,6 +342,9 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
return false;
}
if (shouldIgnoreURL(url)) {
return false;
}
if (Utils.getConfigBoolean("urls_only.save", false)) {
// Output URL to file
Path urlFile = Paths.get(this.workingDir + "/urls.txt");

3
src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java
View File

@@ -160,6 +160,9 @@ public abstract class AbstractJSONRipper extends AbstractRipper {
LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
return false;
}
if (shouldIgnoreURL(url)) {
return false;
}
if (Utils.getConfigBoolean("urls_only.save", false)) {
// Output URL to file
Path urlFile = Paths.get(this.workingDir + "/urls.txt");

14
src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
View File

@@ -699,4 +699,18 @@ public abstract class AbstractRipper
protected boolean useByteProgessBar() { return false;}
// If true ripme will try to resume a broken download for this ripper
protected boolean tryResumeDownload() { return false;}
protected boolean shouldIgnoreURL(URL url) {
final String[] ignoredExtensions = Utils.getConfigStringArray("download.ignore_extensions");
if (ignoredExtensions == null || ignoredExtensions.length == 0) return false; // nothing ignored
String[] pathElements = url.getPath().split("\\.");
if (pathElements.length == 0) return false; // no extension, can't filter
String extension = pathElements[pathElements.length - 1];
for (String ignoredExtension : ignoredExtensions) {
if (ignoredExtension.equalsIgnoreCase(extension)) {
return true;
}
}
return false;
}
}

3
src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java
View File

@@ -70,6 +70,9 @@ public abstract class AlbumRipper extends AbstractRipper {
LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
return false;
}
if (shouldIgnoreURL(url)) {
return false;
}
if (Utils.getConfigBoolean("urls_only.save", false)) {
// Output URL to file
Path urlFile = Paths.get(this.workingDir + "/urls.txt");

3
src/main/java/com/rarchives/ripme/ripper/VideoRipper.java
View File

@@ -68,6 +68,9 @@ public abstract class VideoRipper extends AbstractRipper {
this.url = url;
return true;
}
if (shouldIgnoreURL(url)) {
return false;
}
threadPool.addThread(new DownloadVideoThread(url, saveAs, this));
}
return true;

3
src/main/resources/rip.properties
View File

@@ -17,6 +17,9 @@ page.timeout = 5000
# Maximum size of downloaded files in bytes (required)
download.max_size = 104857600
# Any URLs ending with one of these comma-separated values will be skipped
#download.ignore_extensions = mp4,gif,m4v,webm,html
# Don't retry on 404 errors
error.skip404 = true