From d604d4059659fca6653d70266feaa049b6d7d1b5 Mon Sep 17 00:00:00 2001
From: cyian-1756 <devnull64@vfemail.net>
Date: Sat, 5 May 2018 05:36:00 -0400
Subject: [PATCH] Added option for addURLToDownload which sets the file
extension to the files MIME type
---
.../com/rarchives/ripme/ripper/AbstractRipper.java | 11 ++++++++---
.../com/rarchives/ripme/ripper/AlbumRipper.java | 6 +++---
.../rarchives/ripme/ripper/DownloadFileThread.java | 13 +++++++++++--
.../com/rarchives/ripme/ripper/VideoRipper.java | 3 ++-
.../ripme/ripper/rippers/FivehundredpxRipper.java | 2 +-
.../ripme/ripper/rippers/TsuminoRipper.java | 6 +++++-
6 files changed, 30 insertions(+), 11 deletions(-)
diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
index 4438ae2a..20889495 100644
--- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
@@ -192,7 +192,8 @@ public abstract class AbstractRipper
* True if downloaded successfully
* False if failed to download
*/
- protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String, String> cookies);
+ protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String, String> cookies,
+ Boolean getFileExtFromMIME);
/**
* Queues image to be downloaded and saved.
@@ -212,7 +213,7 @@ public abstract class AbstractRipper
* True if downloaded successfully
* False if failed to download
*/
- protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String, String> cookies, String fileName, String extension) {
+ protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String, String> cookies, String fileName, String extension, Boolean getFileExtFromMIME) {
// Don't re-add the url if it was downloaded in a previous rip
if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) {
if (hasDownloadedURL(url.toExternalForm())) {
@@ -257,7 +258,11 @@ public abstract class AbstractRipper
logger.debug("Unable to write URL history file");
}
}
- return addURLToDownload(url, saveFileAs, referrer, cookies);
+ return addURLToDownload(url, saveFileAs, referrer, cookies, getFileExtFromMIME);
+ }
+
+ protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String,String> cookies, String fileName, String extension) {
+ return addURLToDownload(url, prefix, subdirectory, referrer, cookies, fileName, extension, false);
}
protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String, String> cookies, String fileName) {
diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java
index 1726343a..f700f012 100644
--- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java
@@ -50,7 +50,7 @@ public abstract class AlbumRipper extends AbstractRipper {
/**
* Queues multiple URLs of single images to download from a single Album URL
*/
- public boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies) {
+ public boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies, Boolean getFileExtFromMIME) {
// Only download one file if this is a test.
if (super.isThisATest() &&
(itemsPending.size() > 0 || itemsCompleted.size() > 0 || itemsErrored.size() > 0)) {
@@ -82,7 +82,7 @@ public abstract class AlbumRipper extends AbstractRipper {
}
else {
itemsPending.put(url, saveAs);
- DownloadFileThread dft = new DownloadFileThread(url, saveAs, this);
+ DownloadFileThread dft = new DownloadFileThread(url, saveAs, this, getFileExtFromMIME);
if (referrer != null) {
dft.setReferrer(referrer);
}
@@ -96,7 +96,7 @@ public abstract class AlbumRipper extends AbstractRipper {
@Override
public boolean addURLToDownload(URL url, File saveAs) {
- return addURLToDownload(url, saveAs, null, null);
+ return addURLToDownload(url, saveAs, null, null, false);
}
/**
diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
index c62d58a6..42dedffe 100644
--- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
+++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
@@ -8,6 +8,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
+import java.net.URLConnection;
import java.util.HashMap;
import java.util.Map;
@@ -36,10 +37,11 @@ class DownloadFileThread extends Thread {
private String prettySaveAs;
private AbstractRipper observer;
private int retries;
+ private Boolean getFileExtFromMIME;
private final int TIMEOUT;
- public DownloadFileThread(URL url, File saveAs, AbstractRipper observer) {
+ public DownloadFileThread(URL url, File saveAs, AbstractRipper observer, Boolean getFileExtFromMIME) {
super();
this.url = url;
this.saveAs = saveAs;
@@ -47,6 +49,7 @@ class DownloadFileThread extends Thread {
this.observer = observer;
this.retries = Utils.getConfigInteger("download.retries", 1);
this.TIMEOUT = Utils.getConfigInteger("download.timeout", 60000);
+ this.getFileExtFromMIME = getFileExtFromMIME;
}
public void setReferrer(String referrer) {
@@ -143,9 +146,15 @@ class DownloadFileThread extends Thread {
observer.downloadErrored(url, "Imgur image is 404: " + url.toExternalForm());
return;
}
-
// Save file
bis = new BufferedInputStream(huc.getInputStream());
+
+ // Check if we should get the file ext from the MIME type
+ if (getFileExtFromMIME) {
+ String fileExt = URLConnection.guessContentTypeFromStream(bis).replaceAll("image/", "");
+ saveAs = new File(saveAs.toString() + "." + fileExt);
+ }
+
fos = new FileOutputStream(saveAs);
IOUtils.copy(bis, fos);
break; // Download successful: break out of infinite loop
diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java
index 13008cd9..29200d5a 100644
--- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java
@@ -10,6 +10,7 @@ import java.util.Map;
import com.rarchives.ripme.ui.RipStatusMessage;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Utils;
+import com.sun.org.apache.xpath.internal.operations.Bool;
public abstract class VideoRipper extends AbstractRipper {
@@ -70,7 +71,7 @@ public abstract class VideoRipper extends AbstractRipper {
}
@Override
- public boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies) {
+ public boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies, Boolean getFileExtFromMIME) {
return addURLToDownload(url, saveAs);
}
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java
index 7e532943..93aedba2 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java
@@ -332,7 +332,7 @@ public class FivehundredpxRipper extends AbstractJSONRipper {
String[] fields = u.split("/");
String prefix = getPrefix(index) + fields[fields.length - 3];
File saveAs = new File(getWorkingDir() + File.separator + prefix + ".jpg");
- addURLToDownload(url, saveAs, "", null);
+ addURLToDownload(url, saveAs, "", null, false);
}
}
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java
index 6030c9f0..7d35fc1d 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java
@@ -98,6 +98,10 @@ public class TsuminoRipper extends AbstractHTMLRipper {
@Override
public void downloadURL(URL url, int index) {
sleep(1000);
- addURLToDownload(url, getPrefix(index), "", null, null, null, "png");
+ /*
+ There is no way to tell if an image returned from tsumino.com is a png to jpg. The content-type header is always
+ "image/jpeg" even when the image is a png. The file ext is not included in the url.
+ */
+ addURLToDownload(url, getPrefix(index), "", null, null, null, null, true);
}
}