1
0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-08-12 00:44:03 +02:00

remove warnings in AbstractRipper AbstractHTMLRipper DownloadFileThread Utils

This commit is contained in:
soloturn
2022-04-02 08:33:06 +02:00
parent 9d66f41584
commit 5c9c13abcd
4 changed files with 40 additions and 83 deletions

View File

@@ -309,7 +309,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
*/ */
public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map<String,String> cookies, Boolean getFileExtFromMIME) { public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map<String,String> cookies, Boolean getFileExtFromMIME) {
// Only download one file if this is a test. // Only download one file if this is a test.
if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { if (isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) {
stop(); stop();
itemsPending.clear(); itemsPending.clear();
return false; return false;
@@ -388,7 +388,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
} }
@Override @Override
/** /*
* Cleans up & tells user about failed download. * Cleans up & tells user about failed download.
*/ */
public void downloadErrored(URL url, String reason) { public void downloadErrored(URL url, String reason) {
@@ -436,8 +436,6 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
* Sets directory to save all ripped files to. * Sets directory to save all ripped files to.
* @param url * @param url
* URL to define how the working directory should be saved. * URL to define how the working directory should be saved.
* @throws
* IOException
*/ */
@Override @Override
public void setWorkingDir(URL url) throws IOException { public void setWorkingDir(URL url) throws IOException {
@@ -447,12 +445,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
if (!path.endsWith(File.separator)) { if (!path.endsWith(File.separator)) {
path += File.separator; path += File.separator;
} }
String title; String title = getAlbumTitle(this.url);
if (Utils.getConfigBoolean("album_titles.save", true)) {
title = getAlbumTitle(this.url);
} else {
title = super.getAlbumTitle(this.url);
}
LOGGER.debug("Using album title '" + title + "'"); LOGGER.debug("Using album title '" + title + "'");
title = Utils.filesystemSafe(title); title = Utils.filesystemSafe(title);
@@ -485,12 +478,11 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
*/ */
@Override @Override
public String getStatusText() { public String getStatusText() {
String sb = getCompletionPercentage() + return getCompletionPercentage() +
"% " + "% " +
"- Pending: " + itemsPending.size() + "- Pending: " + itemsPending.size() +
", Completed: " + itemsCompleted.size() + ", Completed: " + itemsCompleted.size() +
", Errored: " + itemsErrored.size(); ", Errored: " + itemsErrored.size();
return sb;
} }

View File

@@ -243,7 +243,7 @@ public abstract class AbstractRipper
*/ */
protected boolean addURLToDownload(URL url, Map<String, String> options, Map<String, String> cookies) { protected boolean addURLToDownload(URL url, Map<String, String> options, Map<String, String> cookies) {
// Bit of a hack but this lets us pass a bool using a map<string,String> // Bit of a hack but this lets us pass a bool using a map<string,String>
boolean useMIME = options.getOrDefault("getFileExtFromMIME", "false").toLowerCase().equals("true"); boolean useMIME = options.getOrDefault("getFileExtFromMIME", "false").equalsIgnoreCase("true");
return addURLToDownload(url, options.getOrDefault("prefix", ""), options.getOrDefault("subdirectory", ""), options.getOrDefault("referrer", null), return addURLToDownload(url, options.getOrDefault("prefix", ""), options.getOrDefault("subdirectory", ""), options.getOrDefault("referrer", null),
cookies, options.getOrDefault("fileName", null), options.getOrDefault("extension", null), useMIME); cookies, options.getOrDefault("fileName", null), options.getOrDefault("extension", null), useMIME);
} }
@@ -455,15 +455,11 @@ public abstract class AbstractRipper
public abstract void downloadCompleted(URL url, Path saveAs); public abstract void downloadCompleted(URL url, Path saveAs);
/** /**
* Notifies observers that a file could not be downloaded (includes a reason). * Notifies observers that a file could not be downloaded (includes a reason).
* @param url
* @param reason
*/ */
public abstract void downloadErrored(URL url, String reason); public abstract void downloadErrored(URL url, String reason);
/** /**
* Notify observers that a download could not be completed, * Notify observers that a download could not be completed,
* but was not technically an "error". * but was not technically an "error".
* @param url
* @param file
*/ */
public abstract void downloadExists(URL url, Path file); public abstract void downloadExists(URL url, Path file);
@@ -581,7 +577,6 @@ public abstract class AbstractRipper
* The package name. * The package name.
* @return * @return
* List of constructors for all eligible Rippers. * List of constructors for all eligible Rippers.
* @throws Exception
*/ */
public static List<Constructor<?>> getRipperConstructors(String pkg) throws Exception { public static List<Constructor<?>> getRipperConstructors(String pkg) throws Exception {
List<Constructor<?>> constructors = new ArrayList<>(); List<Constructor<?>> constructors = new ArrayList<>();
@@ -595,8 +590,7 @@ public abstract class AbstractRipper
/** /**
* Sends an update message to the relevant observer(s) on this ripper. * Sends an update message to the relevant observer(s) on this ripper.
* @param status * @param status
* @param message
*/ */
public void sendUpdate(STATUS status, Object message) { public void sendUpdate(STATUS status, Object message) {
if (observer == null) { if (observer == null) {

View File

@@ -10,11 +10,9 @@ import java.nio.file.Paths;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.ResourceBundle;
import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.HttpsURLConnection;
import com.rarchives.ripme.ui.MainWindow;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.jsoup.HttpStatusException; import org.jsoup.HttpStatusException;
@@ -32,12 +30,12 @@ class DownloadFileThread extends Thread {
private String referrer = ""; private String referrer = "";
private Map<String, String> cookies = new HashMap<>(); private Map<String, String> cookies = new HashMap<>();
private URL url; private final URL url;
private File saveAs; private File saveAs;
private String prettySaveAs; private final String prettySaveAs;
private AbstractRipper observer; private final AbstractRipper observer;
private int retries; private final int retries;
private Boolean getFileExtFromMIME; private final Boolean getFileExtFromMIME;
private final int TIMEOUT; private final int TIMEOUT;
@@ -69,7 +67,7 @@ class DownloadFileThread extends Thread {
saveAs = new File( saveAs = new File(
saveAs.getParentFile().getAbsolutePath() + File.separator + Utils.sanitizeSaveAs(saveAs.getName())); saveAs.getParentFile().getAbsolutePath() + File.separator + Utils.sanitizeSaveAs(saveAs.getName()));
long fileSize = 0; long fileSize = 0;
int bytesTotal = 0; int bytesTotal;
int bytesDownloaded = 0; int bytesDownloaded = 0;
if (saveAs.exists() && observer.tryResumeDownload()) { if (saveAs.exists() && observer.tryResumeDownload()) {
fileSize = saveAs.length(); fileSize = saveAs.length();
@@ -85,7 +83,7 @@ class DownloadFileThread extends Thread {
&& !observer.tryResumeDownload()) { && !observer.tryResumeDownload()) {
if (Utils.getConfigBoolean("file.overwrite", false)) { if (Utils.getConfigBoolean("file.overwrite", false)) {
logger.info("[!] " + Utils.getLocalizedString("deleting.existing.file") + prettySaveAs); logger.info("[!] " + Utils.getLocalizedString("deleting.existing.file") + prettySaveAs);
saveAs.delete(); if (!saveAs.delete()) logger.error("could not delete existing file: " + saveAs.getAbsolutePath());
} else { } else {
logger.info("[!] " + Utils.getLocalizedString("skipping") + " " + url + " -- " logger.info("[!] " + Utils.getLocalizedString("skipping") + " " + url + " -- "
+ Utils.getLocalizedString("file.already.exists") + ": " + prettySaveAs); + Utils.getLocalizedString("file.already.exists") + ": " + prettySaveAs);
@@ -98,8 +96,6 @@ class DownloadFileThread extends Thread {
int tries = 0; // Number of attempts to download int tries = 0; // Number of attempts to download
do { do {
tries += 1; tries += 1;
InputStream bis = null;
OutputStream fos = null;
try { try {
logger.info(" Downloading file: " + urlToDownload + (tries > 0 ? " Retry #" + tries : "")); logger.info(" Downloading file: " + urlToDownload + (tries > 0 ? " Retry #" + tries : ""));
observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm());
@@ -122,14 +118,14 @@ class DownloadFileThread extends Thread {
huc.setRequestProperty("Referer", referrer); // Sic huc.setRequestProperty("Referer", referrer); // Sic
} }
huc.setRequestProperty("User-agent", AbstractRipper.USER_AGENT); huc.setRequestProperty("User-agent", AbstractRipper.USER_AGENT);
String cookie = ""; StringBuilder cookie = new StringBuilder();
for (String key : cookies.keySet()) { for (String key : cookies.keySet()) {
if (!cookie.equals("")) { if (!cookie.toString().equals("")) {
cookie += "; "; cookie.append("; ");
} }
cookie += key + "=" + cookies.get(key); cookie.append(key).append("=").append(cookies.get(key));
} }
huc.setRequestProperty("Cookie", cookie); huc.setRequestProperty("Cookie", cookie.toString());
if (observer.tryResumeDownload()) { if (observer.tryResumeDownload()) {
if (fileSize != 0) { if (fileSize != 0) {
huc.setRequestProperty("Range", "bytes=" + fileSize + "-"); huc.setRequestProperty("Range", "bytes=" + fileSize + "-");
@@ -187,6 +183,7 @@ class DownloadFileThread extends Thread {
} }
// Save file // Save file
InputStream bis;
bis = new BufferedInputStream(huc.getInputStream()); bis = new BufferedInputStream(huc.getInputStream());
// Check if we should get the file ext from the MIME type // Check if we should get the file ext from the MIME type
@@ -212,6 +209,7 @@ class DownloadFileThread extends Thread {
} }
} }
// If we're resuming a download we append data to the existing file // If we're resuming a download we append data to the existing file
OutputStream fos = null;
if (statusCode == 206) { if (statusCode == 206) {
fos = new FileOutputStream(saveAs, true); fos = new FileOutputStream(saveAs, true);
} else { } else {
@@ -240,7 +238,9 @@ class DownloadFileThread extends Thread {
// not allow // not allow
fos = Files.newOutputStream( fos = Files.newOutputStream(
Utils.shortenSaveAsWindows(saveAs.getParentFile().getPath(), saveAs.getName())); Utils.shortenSaveAsWindows(saveAs.getParentFile().getPath(), saveAs.getName()));
assert fos != null: "After shortenSaveAsWindows: " + saveAs.getAbsolutePath();
} }
assert fos != null: e.getStackTrace();
} }
} }
byte[] data = new byte[1024 * 256]; byte[] data = new byte[1024 * 256];
@@ -292,20 +292,6 @@ class DownloadFileThread extends Thread {
Utils.getLocalizedString("failed.to.download") + " " + url.toExternalForm()); Utils.getLocalizedString("failed.to.download") + " " + url.toExternalForm());
return; return;
}finally {
// Close any open streams
try {
if (bis != null) {
bis.close();
}
} catch (IOException e) {
}
try {
if (fos != null) {
fos.close();
}
} catch (IOException e) {
}
} }
if (tries > this.retries) { if (tries > this.retries) {
logger.error("[!] " + Utils.getLocalizedString("exceeded.maximum.retries") + " (" + this.retries logger.error("[!] " + Utils.getLocalizedString("exceeded.maximum.retries") + " (" + this.retries

View File

@@ -20,13 +20,13 @@ import javax.sound.sampled.LineEvent;
import java.io.File; import java.io.File;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.Constructor; import java.lang.reflect.Constructor;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.net.URLDecoder; import java.net.URLDecoder;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystem; import java.nio.file.FileSystem;
import java.nio.file.FileSystems; import java.nio.file.FileSystems;
import java.nio.file.Files; import java.nio.file.Files;
@@ -62,7 +62,7 @@ public class Utils {
private static final int SHORTENED_PATH_LENGTH = 12; private static final int SHORTENED_PATH_LENGTH = 12;
private static PropertiesConfiguration config; private static PropertiesConfiguration config;
private static HashMap<String, HashMap<String, String>> cookieCache; private static final HashMap<String, HashMap<String, String>> cookieCache;
private static final HashMap<ByteBuffer, String> magicHash = new HashMap<>(); private static final HashMap<ByteBuffer, String> magicHash = new HashMap<>();
private static ResourceBundle resourceBundle; private static ResourceBundle resourceBundle;
@@ -256,11 +256,7 @@ public class Utils {
*/ */
private static boolean portableMode() { private static boolean portableMode() {
Path file = getJarDirectory().resolve(CONFIG_FILE); Path file = getJarDirectory().resolve(CONFIG_FILE);
if (Files.exists(file) && !Files.isDirectory(file)) { return Files.exists(file) && !Files.isDirectory(file);
return true;
}
return false;
} }
/** /**
@@ -326,8 +322,7 @@ public class Utils {
* @return saveAs in relation to the CWD * @return saveAs in relation to the CWD
*/ */
public static String removeCWD(Path saveAs) { public static String removeCWD(Path saveAs) {
String prettySaveAs = saveAs.relativize(Paths.get(".").toAbsolutePath()).toString(); return saveAs.relativize(Paths.get(".").toAbsolutePath()).toString();
return prettySaveAs;
} }
/** /**
@@ -409,7 +404,7 @@ public class Utils {
// Load from JAR // Load from JAR
try { try {
String jarPath = fullPath.replaceFirst("[.]jar[!].*", ".jar").replaceFirst("file:", ""); String jarPath = fullPath.replaceFirst("[.]jar[!].*", ".jar").replaceFirst("file:", "");
jarPath = URLDecoder.decode(jarPath, "UTF-8"); jarPath = URLDecoder.decode(jarPath, StandardCharsets.UTF_8);
JarFile jarFile = new JarFile(jarPath); JarFile jarFile = new JarFile(jarPath);
Enumeration<JarEntry> entries = jarFile.entries(); Enumeration<JarEntry> entries = jarFile.entries();
while (entries.hasMoreElements()) { while (entries.hasMoreElements()) {
@@ -662,18 +657,13 @@ public class Utils {
String[] parts = query.split("&"); String[] parts = query.split("&");
int pos; int pos;
try { for (String part : parts) {
for (String part : parts) { if ((pos = part.indexOf('=')) >= 0) {
if ((pos = part.indexOf('=')) >= 0) { res.put(URLDecoder.decode(part.substring(0, pos), StandardCharsets.UTF_8),
res.put(URLDecoder.decode(part.substring(0, pos), "UTF-8"), URLDecoder.decode(part.substring(pos + 1), StandardCharsets.UTF_8));
URLDecoder.decode(part.substring(pos + 1), "UTF-8")); } else {
} else { res.put(URLDecoder.decode(part, StandardCharsets.UTF_8), "");
res.put(URLDecoder.decode(part, "UTF-8"), "");
}
} }
} catch (UnsupportedEncodingException e) {
// Shouldn't happen since UTF-8 is required to be supported
throw new RuntimeException(e);
} }
return res; return res;
@@ -694,20 +684,15 @@ public class Utils {
String[] parts = query.split("&"); String[] parts = query.split("&");
int pos; int pos;
try { for (String part : parts) {
for (String part : parts) { if ((pos = part.indexOf('=')) >= 0) {
if ((pos = part.indexOf('=')) >= 0) { if (URLDecoder.decode(part.substring(0, pos), StandardCharsets.UTF_8).equals(key)) {
if (URLDecoder.decode(part.substring(0, pos), "UTF-8").equals(key)) { return URLDecoder.decode(part.substring(pos + 1), StandardCharsets.UTF_8);
return URLDecoder.decode(part.substring(pos + 1), "UTF-8");
}
} else if (URLDecoder.decode(part, "UTF-8").equals(key)) {
return "";
} }
} else if (URLDecoder.decode(part, StandardCharsets.UTF_8).equals(key)) {
return "";
} }
} catch (UnsupportedEncodingException e) {
// Shouldn't happen since UTF-8 is required to be supported
throw new RuntimeException(e);
} }
return null; return null;