mirror of
https://github.com/RipMeApp/ripme.git
synced 2025-08-12 00:44:03 +02:00
remove warnings in AbstractRipper AbstractHTMLRipper DownloadFileThread Utils
This commit is contained in:
@@ -309,7 +309,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
|
|||||||
*/
|
*/
|
||||||
public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map<String,String> cookies, Boolean getFileExtFromMIME) {
|
public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map<String,String> cookies, Boolean getFileExtFromMIME) {
|
||||||
// Only download one file if this is a test.
|
// Only download one file if this is a test.
|
||||||
if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) {
|
if (isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) {
|
||||||
stop();
|
stop();
|
||||||
itemsPending.clear();
|
itemsPending.clear();
|
||||||
return false;
|
return false;
|
||||||
@@ -388,7 +388,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
/**
|
/*
|
||||||
* Cleans up & tells user about failed download.
|
* Cleans up & tells user about failed download.
|
||||||
*/
|
*/
|
||||||
public void downloadErrored(URL url, String reason) {
|
public void downloadErrored(URL url, String reason) {
|
||||||
@@ -436,8 +436,6 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
|
|||||||
* Sets directory to save all ripped files to.
|
* Sets directory to save all ripped files to.
|
||||||
* @param url
|
* @param url
|
||||||
* URL to define how the working directory should be saved.
|
* URL to define how the working directory should be saved.
|
||||||
* @throws
|
|
||||||
* IOException
|
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public void setWorkingDir(URL url) throws IOException {
|
public void setWorkingDir(URL url) throws IOException {
|
||||||
@@ -447,12 +445,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
|
|||||||
if (!path.endsWith(File.separator)) {
|
if (!path.endsWith(File.separator)) {
|
||||||
path += File.separator;
|
path += File.separator;
|
||||||
}
|
}
|
||||||
String title;
|
String title = getAlbumTitle(this.url);
|
||||||
if (Utils.getConfigBoolean("album_titles.save", true)) {
|
|
||||||
title = getAlbumTitle(this.url);
|
|
||||||
} else {
|
|
||||||
title = super.getAlbumTitle(this.url);
|
|
||||||
}
|
|
||||||
LOGGER.debug("Using album title '" + title + "'");
|
LOGGER.debug("Using album title '" + title + "'");
|
||||||
|
|
||||||
title = Utils.filesystemSafe(title);
|
title = Utils.filesystemSafe(title);
|
||||||
@@ -485,12 +478,11 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
|
|||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public String getStatusText() {
|
public String getStatusText() {
|
||||||
String sb = getCompletionPercentage() +
|
return getCompletionPercentage() +
|
||||||
"% " +
|
"% " +
|
||||||
"- Pending: " + itemsPending.size() +
|
"- Pending: " + itemsPending.size() +
|
||||||
", Completed: " + itemsCompleted.size() +
|
", Completed: " + itemsCompleted.size() +
|
||||||
", Errored: " + itemsErrored.size();
|
", Errored: " + itemsErrored.size();
|
||||||
return sb;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@@ -243,7 +243,7 @@ public abstract class AbstractRipper
|
|||||||
*/
|
*/
|
||||||
protected boolean addURLToDownload(URL url, Map<String, String> options, Map<String, String> cookies) {
|
protected boolean addURLToDownload(URL url, Map<String, String> options, Map<String, String> cookies) {
|
||||||
// Bit of a hack but this lets us pass a bool using a map<string,String>
|
// Bit of a hack but this lets us pass a bool using a map<string,String>
|
||||||
boolean useMIME = options.getOrDefault("getFileExtFromMIME", "false").toLowerCase().equals("true");
|
boolean useMIME = options.getOrDefault("getFileExtFromMIME", "false").equalsIgnoreCase("true");
|
||||||
return addURLToDownload(url, options.getOrDefault("prefix", ""), options.getOrDefault("subdirectory", ""), options.getOrDefault("referrer", null),
|
return addURLToDownload(url, options.getOrDefault("prefix", ""), options.getOrDefault("subdirectory", ""), options.getOrDefault("referrer", null),
|
||||||
cookies, options.getOrDefault("fileName", null), options.getOrDefault("extension", null), useMIME);
|
cookies, options.getOrDefault("fileName", null), options.getOrDefault("extension", null), useMIME);
|
||||||
}
|
}
|
||||||
@@ -455,15 +455,11 @@ public abstract class AbstractRipper
|
|||||||
public abstract void downloadCompleted(URL url, Path saveAs);
|
public abstract void downloadCompleted(URL url, Path saveAs);
|
||||||
/**
|
/**
|
||||||
* Notifies observers that a file could not be downloaded (includes a reason).
|
* Notifies observers that a file could not be downloaded (includes a reason).
|
||||||
* @param url
|
|
||||||
* @param reason
|
|
||||||
*/
|
*/
|
||||||
public abstract void downloadErrored(URL url, String reason);
|
public abstract void downloadErrored(URL url, String reason);
|
||||||
/**
|
/**
|
||||||
* Notify observers that a download could not be completed,
|
* Notify observers that a download could not be completed,
|
||||||
* but was not technically an "error".
|
* but was not technically an "error".
|
||||||
* @param url
|
|
||||||
* @param file
|
|
||||||
*/
|
*/
|
||||||
public abstract void downloadExists(URL url, Path file);
|
public abstract void downloadExists(URL url, Path file);
|
||||||
|
|
||||||
@@ -581,7 +577,6 @@ public abstract class AbstractRipper
|
|||||||
* The package name.
|
* The package name.
|
||||||
* @return
|
* @return
|
||||||
* List of constructors for all eligible Rippers.
|
* List of constructors for all eligible Rippers.
|
||||||
* @throws Exception
|
|
||||||
*/
|
*/
|
||||||
public static List<Constructor<?>> getRipperConstructors(String pkg) throws Exception {
|
public static List<Constructor<?>> getRipperConstructors(String pkg) throws Exception {
|
||||||
List<Constructor<?>> constructors = new ArrayList<>();
|
List<Constructor<?>> constructors = new ArrayList<>();
|
||||||
@@ -595,8 +590,7 @@ public abstract class AbstractRipper
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Sends an update message to the relevant observer(s) on this ripper.
|
* Sends an update message to the relevant observer(s) on this ripper.
|
||||||
* @param status
|
* @param status
|
||||||
* @param message
|
|
||||||
*/
|
*/
|
||||||
public void sendUpdate(STATUS status, Object message) {
|
public void sendUpdate(STATUS status, Object message) {
|
||||||
if (observer == null) {
|
if (observer == null) {
|
||||||
|
@@ -10,11 +10,9 @@ import java.nio.file.Paths;
|
|||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.ResourceBundle;
|
|
||||||
|
|
||||||
import javax.net.ssl.HttpsURLConnection;
|
import javax.net.ssl.HttpsURLConnection;
|
||||||
|
|
||||||
import com.rarchives.ripme.ui.MainWindow;
|
|
||||||
import org.apache.logging.log4j.LogManager;
|
import org.apache.logging.log4j.LogManager;
|
||||||
import org.apache.logging.log4j.Logger;
|
import org.apache.logging.log4j.Logger;
|
||||||
import org.jsoup.HttpStatusException;
|
import org.jsoup.HttpStatusException;
|
||||||
@@ -32,12 +30,12 @@ class DownloadFileThread extends Thread {
|
|||||||
private String referrer = "";
|
private String referrer = "";
|
||||||
private Map<String, String> cookies = new HashMap<>();
|
private Map<String, String> cookies = new HashMap<>();
|
||||||
|
|
||||||
private URL url;
|
private final URL url;
|
||||||
private File saveAs;
|
private File saveAs;
|
||||||
private String prettySaveAs;
|
private final String prettySaveAs;
|
||||||
private AbstractRipper observer;
|
private final AbstractRipper observer;
|
||||||
private int retries;
|
private final int retries;
|
||||||
private Boolean getFileExtFromMIME;
|
private final Boolean getFileExtFromMIME;
|
||||||
|
|
||||||
private final int TIMEOUT;
|
private final int TIMEOUT;
|
||||||
|
|
||||||
@@ -69,7 +67,7 @@ class DownloadFileThread extends Thread {
|
|||||||
saveAs = new File(
|
saveAs = new File(
|
||||||
saveAs.getParentFile().getAbsolutePath() + File.separator + Utils.sanitizeSaveAs(saveAs.getName()));
|
saveAs.getParentFile().getAbsolutePath() + File.separator + Utils.sanitizeSaveAs(saveAs.getName()));
|
||||||
long fileSize = 0;
|
long fileSize = 0;
|
||||||
int bytesTotal = 0;
|
int bytesTotal;
|
||||||
int bytesDownloaded = 0;
|
int bytesDownloaded = 0;
|
||||||
if (saveAs.exists() && observer.tryResumeDownload()) {
|
if (saveAs.exists() && observer.tryResumeDownload()) {
|
||||||
fileSize = saveAs.length();
|
fileSize = saveAs.length();
|
||||||
@@ -85,7 +83,7 @@ class DownloadFileThread extends Thread {
|
|||||||
&& !observer.tryResumeDownload()) {
|
&& !observer.tryResumeDownload()) {
|
||||||
if (Utils.getConfigBoolean("file.overwrite", false)) {
|
if (Utils.getConfigBoolean("file.overwrite", false)) {
|
||||||
logger.info("[!] " + Utils.getLocalizedString("deleting.existing.file") + prettySaveAs);
|
logger.info("[!] " + Utils.getLocalizedString("deleting.existing.file") + prettySaveAs);
|
||||||
saveAs.delete();
|
if (!saveAs.delete()) logger.error("could not delete existing file: " + saveAs.getAbsolutePath());
|
||||||
} else {
|
} else {
|
||||||
logger.info("[!] " + Utils.getLocalizedString("skipping") + " " + url + " -- "
|
logger.info("[!] " + Utils.getLocalizedString("skipping") + " " + url + " -- "
|
||||||
+ Utils.getLocalizedString("file.already.exists") + ": " + prettySaveAs);
|
+ Utils.getLocalizedString("file.already.exists") + ": " + prettySaveAs);
|
||||||
@@ -98,8 +96,6 @@ class DownloadFileThread extends Thread {
|
|||||||
int tries = 0; // Number of attempts to download
|
int tries = 0; // Number of attempts to download
|
||||||
do {
|
do {
|
||||||
tries += 1;
|
tries += 1;
|
||||||
InputStream bis = null;
|
|
||||||
OutputStream fos = null;
|
|
||||||
try {
|
try {
|
||||||
logger.info(" Downloading file: " + urlToDownload + (tries > 0 ? " Retry #" + tries : ""));
|
logger.info(" Downloading file: " + urlToDownload + (tries > 0 ? " Retry #" + tries : ""));
|
||||||
observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm());
|
observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm());
|
||||||
@@ -122,14 +118,14 @@ class DownloadFileThread extends Thread {
|
|||||||
huc.setRequestProperty("Referer", referrer); // Sic
|
huc.setRequestProperty("Referer", referrer); // Sic
|
||||||
}
|
}
|
||||||
huc.setRequestProperty("User-agent", AbstractRipper.USER_AGENT);
|
huc.setRequestProperty("User-agent", AbstractRipper.USER_AGENT);
|
||||||
String cookie = "";
|
StringBuilder cookie = new StringBuilder();
|
||||||
for (String key : cookies.keySet()) {
|
for (String key : cookies.keySet()) {
|
||||||
if (!cookie.equals("")) {
|
if (!cookie.toString().equals("")) {
|
||||||
cookie += "; ";
|
cookie.append("; ");
|
||||||
}
|
}
|
||||||
cookie += key + "=" + cookies.get(key);
|
cookie.append(key).append("=").append(cookies.get(key));
|
||||||
}
|
}
|
||||||
huc.setRequestProperty("Cookie", cookie);
|
huc.setRequestProperty("Cookie", cookie.toString());
|
||||||
if (observer.tryResumeDownload()) {
|
if (observer.tryResumeDownload()) {
|
||||||
if (fileSize != 0) {
|
if (fileSize != 0) {
|
||||||
huc.setRequestProperty("Range", "bytes=" + fileSize + "-");
|
huc.setRequestProperty("Range", "bytes=" + fileSize + "-");
|
||||||
@@ -187,6 +183,7 @@ class DownloadFileThread extends Thread {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Save file
|
// Save file
|
||||||
|
InputStream bis;
|
||||||
bis = new BufferedInputStream(huc.getInputStream());
|
bis = new BufferedInputStream(huc.getInputStream());
|
||||||
|
|
||||||
// Check if we should get the file ext from the MIME type
|
// Check if we should get the file ext from the MIME type
|
||||||
@@ -212,6 +209,7 @@ class DownloadFileThread extends Thread {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
// If we're resuming a download we append data to the existing file
|
// If we're resuming a download we append data to the existing file
|
||||||
|
OutputStream fos = null;
|
||||||
if (statusCode == 206) {
|
if (statusCode == 206) {
|
||||||
fos = new FileOutputStream(saveAs, true);
|
fos = new FileOutputStream(saveAs, true);
|
||||||
} else {
|
} else {
|
||||||
@@ -240,7 +238,9 @@ class DownloadFileThread extends Thread {
|
|||||||
// not allow
|
// not allow
|
||||||
fos = Files.newOutputStream(
|
fos = Files.newOutputStream(
|
||||||
Utils.shortenSaveAsWindows(saveAs.getParentFile().getPath(), saveAs.getName()));
|
Utils.shortenSaveAsWindows(saveAs.getParentFile().getPath(), saveAs.getName()));
|
||||||
|
assert fos != null: "After shortenSaveAsWindows: " + saveAs.getAbsolutePath();
|
||||||
}
|
}
|
||||||
|
assert fos != null: e.getStackTrace();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
byte[] data = new byte[1024 * 256];
|
byte[] data = new byte[1024 * 256];
|
||||||
@@ -292,20 +292,6 @@ class DownloadFileThread extends Thread {
|
|||||||
Utils.getLocalizedString("failed.to.download") + " " + url.toExternalForm());
|
Utils.getLocalizedString("failed.to.download") + " " + url.toExternalForm());
|
||||||
return;
|
return;
|
||||||
|
|
||||||
}finally {
|
|
||||||
// Close any open streams
|
|
||||||
try {
|
|
||||||
if (bis != null) {
|
|
||||||
bis.close();
|
|
||||||
}
|
|
||||||
} catch (IOException e) {
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
if (fos != null) {
|
|
||||||
fos.close();
|
|
||||||
}
|
|
||||||
} catch (IOException e) {
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (tries > this.retries) {
|
if (tries > this.retries) {
|
||||||
logger.error("[!] " + Utils.getLocalizedString("exceeded.maximum.retries") + " (" + this.retries
|
logger.error("[!] " + Utils.getLocalizedString("exceeded.maximum.retries") + " (" + this.retries
|
||||||
|
@@ -20,13 +20,13 @@ import javax.sound.sampled.LineEvent;
|
|||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.UnsupportedEncodingException;
|
|
||||||
import java.lang.reflect.Constructor;
|
import java.lang.reflect.Constructor;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.URISyntaxException;
|
import java.net.URISyntaxException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.net.URLDecoder;
|
import java.net.URLDecoder;
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.nio.file.FileSystem;
|
import java.nio.file.FileSystem;
|
||||||
import java.nio.file.FileSystems;
|
import java.nio.file.FileSystems;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
@@ -62,7 +62,7 @@ public class Utils {
|
|||||||
private static final int SHORTENED_PATH_LENGTH = 12;
|
private static final int SHORTENED_PATH_LENGTH = 12;
|
||||||
|
|
||||||
private static PropertiesConfiguration config;
|
private static PropertiesConfiguration config;
|
||||||
private static HashMap<String, HashMap<String, String>> cookieCache;
|
private static final HashMap<String, HashMap<String, String>> cookieCache;
|
||||||
private static final HashMap<ByteBuffer, String> magicHash = new HashMap<>();
|
private static final HashMap<ByteBuffer, String> magicHash = new HashMap<>();
|
||||||
|
|
||||||
private static ResourceBundle resourceBundle;
|
private static ResourceBundle resourceBundle;
|
||||||
@@ -256,11 +256,7 @@ public class Utils {
|
|||||||
*/
|
*/
|
||||||
private static boolean portableMode() {
|
private static boolean portableMode() {
|
||||||
Path file = getJarDirectory().resolve(CONFIG_FILE);
|
Path file = getJarDirectory().resolve(CONFIG_FILE);
|
||||||
if (Files.exists(file) && !Files.isDirectory(file)) {
|
return Files.exists(file) && !Files.isDirectory(file);
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -326,8 +322,7 @@ public class Utils {
|
|||||||
* @return saveAs in relation to the CWD
|
* @return saveAs in relation to the CWD
|
||||||
*/
|
*/
|
||||||
public static String removeCWD(Path saveAs) {
|
public static String removeCWD(Path saveAs) {
|
||||||
String prettySaveAs = saveAs.relativize(Paths.get(".").toAbsolutePath()).toString();
|
return saveAs.relativize(Paths.get(".").toAbsolutePath()).toString();
|
||||||
return prettySaveAs;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -409,7 +404,7 @@ public class Utils {
|
|||||||
// Load from JAR
|
// Load from JAR
|
||||||
try {
|
try {
|
||||||
String jarPath = fullPath.replaceFirst("[.]jar[!].*", ".jar").replaceFirst("file:", "");
|
String jarPath = fullPath.replaceFirst("[.]jar[!].*", ".jar").replaceFirst("file:", "");
|
||||||
jarPath = URLDecoder.decode(jarPath, "UTF-8");
|
jarPath = URLDecoder.decode(jarPath, StandardCharsets.UTF_8);
|
||||||
JarFile jarFile = new JarFile(jarPath);
|
JarFile jarFile = new JarFile(jarPath);
|
||||||
Enumeration<JarEntry> entries = jarFile.entries();
|
Enumeration<JarEntry> entries = jarFile.entries();
|
||||||
while (entries.hasMoreElements()) {
|
while (entries.hasMoreElements()) {
|
||||||
@@ -662,18 +657,13 @@ public class Utils {
|
|||||||
String[] parts = query.split("&");
|
String[] parts = query.split("&");
|
||||||
int pos;
|
int pos;
|
||||||
|
|
||||||
try {
|
for (String part : parts) {
|
||||||
for (String part : parts) {
|
if ((pos = part.indexOf('=')) >= 0) {
|
||||||
if ((pos = part.indexOf('=')) >= 0) {
|
res.put(URLDecoder.decode(part.substring(0, pos), StandardCharsets.UTF_8),
|
||||||
res.put(URLDecoder.decode(part.substring(0, pos), "UTF-8"),
|
URLDecoder.decode(part.substring(pos + 1), StandardCharsets.UTF_8));
|
||||||
URLDecoder.decode(part.substring(pos + 1), "UTF-8"));
|
} else {
|
||||||
} else {
|
res.put(URLDecoder.decode(part, StandardCharsets.UTF_8), "");
|
||||||
res.put(URLDecoder.decode(part, "UTF-8"), "");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} catch (UnsupportedEncodingException e) {
|
|
||||||
// Shouldn't happen since UTF-8 is required to be supported
|
|
||||||
throw new RuntimeException(e);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
@@ -694,20 +684,15 @@ public class Utils {
|
|||||||
String[] parts = query.split("&");
|
String[] parts = query.split("&");
|
||||||
int pos;
|
int pos;
|
||||||
|
|
||||||
try {
|
for (String part : parts) {
|
||||||
for (String part : parts) {
|
if ((pos = part.indexOf('=')) >= 0) {
|
||||||
if ((pos = part.indexOf('=')) >= 0) {
|
if (URLDecoder.decode(part.substring(0, pos), StandardCharsets.UTF_8).equals(key)) {
|
||||||
if (URLDecoder.decode(part.substring(0, pos), "UTF-8").equals(key)) {
|
return URLDecoder.decode(part.substring(pos + 1), StandardCharsets.UTF_8);
|
||||||
return URLDecoder.decode(part.substring(pos + 1), "UTF-8");
|
|
||||||
}
|
|
||||||
|
|
||||||
} else if (URLDecoder.decode(part, "UTF-8").equals(key)) {
|
|
||||||
return "";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
} else if (URLDecoder.decode(part, StandardCharsets.UTF_8).equals(key)) {
|
||||||
|
return "";
|
||||||
}
|
}
|
||||||
} catch (UnsupportedEncodingException e) {
|
|
||||||
// Shouldn't happen since UTF-8 is required to be supported
|
|
||||||
throw new RuntimeException(e);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
|
Reference in New Issue
Block a user