From d236d4e417d59cfb4ab63ff836751bca26a0b57c Mon Sep 17 00:00:00 2001 From: MetaPrime Date: Mon, 6 Jan 2025 01:23:16 -0800 Subject: [PATCH] AbstractRipper.java - Fix #2051 by suppressing deprecation warning on java.util.Observable as discussed; light code review, linter fixes, and auto-format. --- .../ripme/ripper/AbstractRipper.java | 371 ++++++++++-------- 1 file changed, 206 insertions(+), 165 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index f290e72b..bb402620 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -21,11 +21,10 @@ import java.util.Observable; import java.util.Scanner; import java.util.concurrent.atomic.AtomicBoolean; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; - import org.jsoup.HttpStatusException; + import com.rarchives.ripme.App; import com.rarchives.ripme.ui.RipStatusComplete; import com.rarchives.ripme.ui.RipStatusHandler; @@ -33,15 +32,21 @@ import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; +// Suppress warning for specifically Observable. Hopefully no other deprecations +// get hidden by this suppression. +// The reason for this is that the deprecation is due to insufficiently powerful +// design. However, it's good enough for us and getting rid of the warning means +// adding our own Observer pattern implementation that is essentially a copy- +// paste of the one in the JDK that has been deprecated. No need to do that. +@SuppressWarnings("deprecation") public abstract class AbstractRipper - extends Observable - implements RipperInterface, Runnable { + extends Observable + implements RipperInterface, Runnable { protected static final Logger LOGGER = LogManager.getLogger(AbstractRipper.class); private final String URLHistoryFile = Utils.getURLHistoryFile(); - public static final String USER_AGENT = - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"; + public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"; protected URL url; protected File workingDir; @@ -51,9 +56,15 @@ public abstract class AbstractRipper private boolean completed = true; public abstract void rip() throws IOException, URISyntaxException; + public abstract String getHost(); + public abstract String getGID(URL url) throws MalformedURLException, URISyntaxException; - public boolean hasASAPRipping() { return false; } + + public boolean hasASAPRipping() { + return false; + } + // Everytime addUrlToDownload skips a already downloaded url this increases by 1 public int alreadyDownloadedUrls = 0; private final AtomicBoolean shouldStop = new AtomicBoolean(false); @@ -63,18 +74,20 @@ public abstract class AbstractRipper LOGGER.trace("stop()"); shouldStop.set(true); } + public boolean isStopped() { return shouldStop.get(); } + protected void stopCheck() throws IOException { if (shouldStop.get()) { throw new IOException("Ripping interrupted"); } } - /** * Adds a URL to the url history file + * * @param downloadedURL URL to check if downloaded */ protected void writeDownloadedURL(String downloadedURL) throws IOException { @@ -125,21 +138,21 @@ public abstract class AbstractRipper } } - /** * Normalize a URL + * * @param url URL to check if downloaded */ public String normalizeUrl(String url) { return url; } - + /** * Checks to see if Ripme has already downloaded a URL + * * @param url URL to check if downloaded - * @return - * Returns true if previously downloaded. - * Returns false if not yet downloaded. + * @return Returns true if previously downloaded. + * Returns false if not yet downloaded. */ protected boolean hasDownloadedURL(String url) { File file = new File(URLHistoryFile); @@ -159,15 +172,12 @@ public abstract class AbstractRipper return false; } - /** * Ensures inheriting ripper can rip this URL, raises exception if not. * Otherwise initializes working directory and thread pool. * - * @param url - * URL to rip. - * @throws IOException - * If anything goes wrong. + * @param url URL to rip. + * @throws IOException If anything goes wrong. */ public AbstractRipper(URL url) throws IOException { if (!canRip(url)) { @@ -182,24 +192,25 @@ public abstract class AbstractRipper /** * Sets ripper's: - * Working directory - * Logger (for debugging) - * FileAppender - * Threadpool - * @throws IOException - * Always be prepared. + * - Working directory + * - Logger (for debugging) + * - FileAppender + * - Threadpool + * + * @throws IOException Always be prepared. */ public void setup() throws IOException, URISyntaxException { setWorkingDir(this.url); - // we do not care if the rollingfileappender is active, just change the logfile in case + // we do not care if the RollingFileAppender is active, + // just change the logfile in case. // TODO this does not work - not even with - // .withFileName("${sys:logFilename}") + // .withFileName("${sys:logFilename}") // in Utils.java, RollingFileAppender. -// System.setProperty("logFilename", this.workingDir + "/log.txt"); -// LOGGER.debug("Changing log file to '{}/log.txt'", this.workingDir); -// LoggerContext ctx = (LoggerContext) LogManager.getContext(false); -// ctx.reconfigure(); -// ctx.updateLoggers(); + // System.setProperty("logFilename", this.workingDir + "/log.txt"); + // LOGGER.debug("Changing log file to '{}/log.txt'", this.workingDir); + // LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + // ctx.reconfigure(); + // ctx.updateLoggers(); this.threadPool = new DownloadThreadPool(); } @@ -210,65 +221,63 @@ public abstract class AbstractRipper /** * Queues image to be downloaded and saved. - * @param url - * URL of the file - * @param saveAs - * Path of the local file to save the content to. + * + * @param url URL of the file + * @param saveAs Path of the local file to save the content to. * @return True on success, false on failure. */ public abstract boolean addURLToDownload(URL url, Path saveAs); /** * Queues image to be downloaded and saved. - * @param url - * URL of the file - * @param saveAs - * Path of the local file to save the content to. - * @param referrer - * The HTTP referrer to use while downloading this file. - * @param cookies - * The cookies to send to the server while downloading this file. - * @return - * True if downloaded successfully - * False if failed to download + * + * @param url URL of the file + * @param saveAs Path of the local file to save the content to. + * @param referrer The HTTP referrer to use while downloading this file. + * @param cookies The cookies to send to the server while downloading this + * file. + * @return True if downloaded successfully + * False if failed to download */ protected abstract boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, - Boolean getFileExtFromMIME); + Boolean getFileExtFromMIME); /** * Queues image to be downloaded and saved. - * @param url - * URL of the file - * @param options - * A map containing any changes to the default options. - * Options are getFileExtFromMIME, prefix, subdirectory, referrer, fileName, extension, getFileExtFromMIME. - * getFileExtFromMIME should be "true" or "false" - * @param cookies - * The cookies to send to the server while downloading this file. - * @return - * True if downloaded successfully - * False if failed to download + * + * @param url URL of the file + * @param options A map containing any changes to the default + * options. + * Options are getFileExtFromMIME, prefix, subdirectory, + * referrer, fileName, extension, getFileExtFromMIME. + * getFileExtFromMIME should be "true" or "false" + * @param cookies The cookies to send to the server while downloading this file. + * @return True if downloaded successfully + * False if failed to download */ protected boolean addURLToDownload(URL url, Map options, Map cookies) { // Bit of a hack but this lets us pass a bool using a map boolean useMIME = options.getOrDefault("getFileExtFromMIME", "false").equalsIgnoreCase("true"); - return addURLToDownload(url, options.getOrDefault("subdirectory", ""), options.getOrDefault("referrer", null), cookies, - options.getOrDefault("prefix", ""), options.getOrDefault("fileName", null), options.getOrDefault("extension", null), + return addURLToDownload(url, + options.getOrDefault("subdirectory", ""), + options.getOrDefault("referrer", null), + cookies, + options.getOrDefault("prefix", ""), options.getOrDefault("fileName", null), + options.getOrDefault("extension", null), useMIME); } - /** * Queues image to be downloaded and saved. - * @param url - * URL of the file - * @param options - * A map containing any changes to the default options. - * Options are getFileExtFromMIME, prefix, subdirectory, referrer, fileName, extension, getFileExtFromMIME. - * getFileExtFromMIME should be "true" or "false" - * @return - * True if downloaded successfully - * False if failed to download + * + * @param url URL of the file + * @param options A map containing any changes to the default + * options. + * Options are getFileExtFromMIME, prefix, subdirectory, + * referrer, fileName, extension, getFileExtFromMIME. + * getFileExtFromMIME should be "true" or "false" + * @return True if downloaded successfully + * False if failed to download */ protected boolean addURLToDownload(URL url, Map options) { return addURLToDownload(url, options, null); @@ -276,32 +285,32 @@ public abstract class AbstractRipper /** * Queues image to be downloaded and saved. - * @param url - * URL of the file - * @param prefix - * Prefix for the downloaded file - * @param subdirectory - * Path to get to desired directory from working directory - * @param referrer - * The HTTP referrer to use while downloading this file. - * @param cookies - * The cookies to send to the server while downloading this file. - * @param fileName - * The name that file will be written to - * @return - * True if downloaded successfully - * False if failed to download + * + * @param url URL of the file + * @param prefix Prefix for the downloaded file + * @param subdirectory Path to get to desired directory from working directory + * @param referrer The HTTP referrer to use while downloading this file. + * @param cookies The cookies to send to the server while downloading this + * file. + * @param fileName The name that file will be written to + * @return True if downloaded successfully + * False if failed to download */ - protected boolean addURLToDownload(URL url, String subdirectory, String referrer, Map cookies, String prefix, String fileName, String extension, Boolean getFileExtFromMIME) { - // A common bug is rippers adding urls that are just "http:". This rejects said urls + protected boolean addURLToDownload(URL url, String subdirectory, String referrer, Map cookies, + String prefix, String fileName, String extension, Boolean getFileExtFromMIME) { + // A common bug is rippers adding urls that are just "http:". + // This rejects said urls. if (url.toExternalForm().equals("http:") || url.toExternalForm().equals("https:")) { - LOGGER.info(url.toExternalForm() + " is a invalid url amd will be changed"); + LOGGER.info(url.toExternalForm() + " is a invalid url and will be changed"); return false; } - // Make sure the url doesn't contain any spaces as that can cause a 400 error when requesting the file + + // Make sure the url doesn't contain any spaces as that can cause a 400 error + // when requesting the file if (url.toExternalForm().contains(" ")) { - // If for some reason the url with all spaces encoded as %20 is malformed print an error + // If for some reason the url with all spaces encoded as %20 is malformed print + // an error try { url = new URI(url.toExternalForm().replaceAll(" ", "%20")).toURL(); } catch (MalformedURLException | URISyntaxException e) { @@ -309,6 +318,7 @@ public abstract class AbstractRipper e.printStackTrace(); } } + // Don't re-add the url if it was downloaded in a previous rip if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) { if (hasDownloadedURL(url.toExternalForm())) { @@ -317,13 +327,17 @@ public abstract class AbstractRipper return false; } } + try { stopCheck(); } catch (IOException e) { LOGGER.debug("Ripper has been stopped"); return false; } - LOGGER.debug("url: " + url + ", subdirectory" + subdirectory + ", referrer: " + referrer + ", cookies: " + cookies + ", prefix: " + prefix + ", fileName: " + fileName); + + LOGGER.debug("url: " + url + ", subdirectory" + subdirectory + ", referrer: " + referrer + ", cookies: " + + cookies + ", prefix: " + prefix + ", fileName: " + fileName); + Path saveAs; try { saveAs = getFilePath(url, subdirectory, prefix, fileName, extension); @@ -336,6 +350,7 @@ public abstract class AbstractRipper LOGGER.error("[!] Error creating save file path for URL '" + url + "':", e); return false; } + if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) { LOGGER.info("Writing " + url.toExternalForm() + " to file"); try { @@ -344,42 +359,46 @@ public abstract class AbstractRipper LOGGER.debug("Unable to write URL history file"); } } + return addURLToDownload(url, saveAs, referrer, cookies, getFileExtFromMIME); } - protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName, String extension) { + protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, + Map cookies, String fileName, String extension) { return addURLToDownload(url, subdirectory, referrer, cookies, prefix, fileName, extension, false); } - protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName) { + protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, + Map cookies, String fileName) { return addURLToDownload(url, prefix, subdirectory, referrer, cookies, fileName, null); } /** * Queues file to be downloaded and saved. With options. - * @param url - * URL to download. - * @param prefix - * Prefix to prepend to the saved filename. - * @param subdirectory - * Sub-directory of the working directory to save the images to. + * + * @param url URL to download. + * @param prefix Prefix to prepend to the saved filename. + * @param subdirectory Sub-directory of the working directory to save the images + * to. * @return True on success, flase on failure. */ protected boolean addURLToDownload(URL url, String prefix, String subdirectory) { return addURLToDownload(url, prefix, subdirectory, null, null, null); } - protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies) { + protected boolean addURLToDownload(URL url, String prefix, String subdirectory, + String referrer, Map cookies) { return addURLToDownload(url, prefix, subdirectory, referrer, cookies, null); } /** * Queues image to be downloaded and saved. * Uses filename from URL (and 'prefix') to decide filename. + * * @param url - * URL to download + * URL to download * @param prefix - * Text to append to saved filename. + * Text to append to saved filename. * @return True on success, flase on failure. */ protected boolean addURLToDownload(URL url, String prefix) { @@ -387,16 +406,19 @@ public abstract class AbstractRipper return addURLToDownload(url, prefix, ""); } - public Path getFilePath(URL url, String subdir, String prefix, String fileName, String extension) throws IOException { + public Path getFilePath(URL url, String subdir, String prefix, String fileName, String extension) + throws IOException { // construct the path: workingdir + subdir + prefix + filename + extension // save into working dir Path filepath = Paths.get(workingDir.getCanonicalPath()); - if (null != App.stringToAppendToFoldername) + if (null != App.stringToAppendToFoldername) { filepath = filepath.resolveSibling(filepath.getFileName() + App.stringToAppendToFoldername); + } - if (null != subdir && !subdir.trim().isEmpty()) + if (null != subdir && !subdir.trim().isEmpty()) { filepath = filepath.resolve(Utils.filesystemSafe(subdir)); + } filepath = filepath.resolve(getFileName(url, prefix, fileName, extension)); return filepath; @@ -406,12 +428,24 @@ public abstract class AbstractRipper // retrieve filename from URL if not passed if (fileName == null || fileName.trim().isEmpty()) { fileName = url.toExternalForm(); - fileName = fileName.substring(fileName.lastIndexOf('/')+1); + fileName = fileName.substring(fileName.lastIndexOf('/') + 1); + } + + if (fileName.indexOf('?') >= 0) { + fileName = fileName.substring(0, fileName.indexOf('?')); + } + + if (fileName.indexOf('#') >= 0) { + fileName = fileName.substring(0, fileName.indexOf('#')); + } + + if (fileName.indexOf('&') >= 0) { + fileName = fileName.substring(0, fileName.indexOf('&')); + } + + if (fileName.indexOf(':') >= 0) { + fileName = fileName.substring(0, fileName.indexOf(':')); } - if (fileName.indexOf('?') >= 0) { fileName = fileName.substring(0, fileName.indexOf('?')); } - if (fileName.indexOf('#') >= 0) { fileName = fileName.substring(0, fileName.indexOf('#')); } - if (fileName.indexOf('&') >= 0) { fileName = fileName.substring(0, fileName.indexOf('&')); } - if (fileName.indexOf(':') >= 0) { fileName = fileName.substring(0, fileName.indexOf(':')); } // add prefix if (prefix != null && !prefix.trim().isEmpty()) { @@ -428,15 +462,16 @@ public abstract class AbstractRipper extension = lastBit[lastBit.length - 1]; } } + // if extension is passed or found, add it if (extension != null) { fileName = fileName + "." + extension; } + // make sure filename is not too long and has no unsupported chars return Utils.sanitizeSaveAs(fileName); } - /** * Waits for downloading threads to complete. */ @@ -449,8 +484,8 @@ public abstract class AbstractRipper /** * Notifies observers that source is being retrieved. - * @param url - * URL being retrieved + * + * @param url URL being retrieved */ public void retrievingSource(String url) { RipStatusMessage msg = new RipStatusMessage(STATUS.LOADING_RESOURCE, url); @@ -461,16 +496,17 @@ public abstract class AbstractRipper /** * Notifies observers that a file download has completed. - * @param url - * URL that was completed. - * @param saveAs - * Where the downloaded file is stored. + * + * @param url URL that was completed. + * @param saveAs Where the downloaded file is stored. */ public abstract void downloadCompleted(URL url, Path saveAs); + /** * Notifies observers that a file could not be downloaded (includes a reason). */ public abstract void downloadErrored(URL url, String reason); + /** * Notify observers that a download could not be completed, * but was not technically an "error". @@ -501,12 +537,13 @@ public abstract class AbstractRipper RipStatusMessage msg = new RipStatusMessage(STATUS.RIP_COMPLETE, rsc); observer.update(this, msg); - // we do not care if the rollingfileappender is active, just change the logfile in case + // we do not care if the rollingfileappender is active, + // just change the logfile in case // TODO - does not work. -// System.setProperty("logFilename", "ripme.log"); -// LOGGER.debug("Changing log file back to 'ripme.log'"); -// LoggerContext ctx = (LoggerContext) LogManager.getContext(false); -// ctx.reconfigure(); + // System.setProperty("logFilename", "ripme.log"); + // LOGGER.debug("Changing log file back to 'ripme.log'"); + // LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + // ctx.reconfigure(); if (Utils.getConfigBoolean("urls_only.save", false)) { String urlFile = this.workingDir + File.separator + "urls.txt"; @@ -521,17 +558,16 @@ public abstract class AbstractRipper /** * Gets URL - * @return - * Returns URL that wants to be downloaded. + * + * @return Returns URL that wants to be downloaded. */ public URL getURL() { return url; } /** - * @return - * Path to the directory in which all files - * ripped via this ripper will be stored. + * @return Path to the directory in which all files + * ripped via this ripper will be stored. */ public File getWorkingDir() { return workingDir; @@ -541,15 +577,11 @@ public abstract class AbstractRipper public abstract void setWorkingDir(URL url) throws IOException, URISyntaxException; /** - * - * @param url - * The URL you want to get the title of. - * @return - * host_URLid - * e.g. (for a reddit post) - * reddit_post_7mg2ur - * @throws MalformedURLException - * If any of those damned URLs gets malformed. + * @param url The URL you want to get the title of. + * @return host_URLid + * e.g. (for a reddit post) + * reddit_post_7mg2ur + * @throws MalformedURLException If any of those damned URLs gets malformed. */ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { try { @@ -561,17 +593,16 @@ public abstract class AbstractRipper /** * Finds, instantiates, and returns a compatible ripper for given URL. - * @param url - * URL to rip. - * @return - * Instantiated ripper ready to rip given URL. - * @throws Exception - * If no compatible rippers can be found. + * + * @param url URL to rip. + * @return Instantiated ripper ready to rip given URL. + * @throws Exception If no compatible rippers can be found. */ public static AbstractRipper getRipper(URL url) throws Exception { for (Constructor constructor : getRipperConstructors("com.rarchives.ripme.ripper.rippers")) { try { - AbstractRipper ripper = (AbstractRipper) constructor.newInstance(url); // by design: can throw ClassCastException + // by design: can throw ClassCastException + AbstractRipper ripper = (AbstractRipper) constructor.newInstance(url); LOGGER.debug("Found album ripper: " + ripper.getClass().getName()); return ripper; } catch (Exception e) { @@ -580,7 +611,8 @@ public abstract class AbstractRipper } for (Constructor constructor : getRipperConstructors("com.rarchives.ripme.ripper.rippers.video")) { try { - VideoRipper ripper = (VideoRipper) constructor.newInstance(url); // by design: can throw ClassCastException + // by design: can throw ClassCastException + VideoRipper ripper = (VideoRipper) constructor.newInstance(url); LOGGER.debug("Found video ripper: " + ripper.getClass().getName()); return ripper; } catch (Exception e) { @@ -591,10 +623,8 @@ public abstract class AbstractRipper } /** - * @param pkg - * The package name. - * @return - * List of constructors for all eligible Rippers. + * @param pkg The package name. + * @return List of constructors for all eligible Rippers. */ public static List> getRipperConstructors(String pkg) throws Exception { List> constructors = new ArrayList<>(); @@ -608,6 +638,7 @@ public abstract class AbstractRipper /** * Sends an update message to the relevant observer(s) on this ripper. + * * @param status */ public void sendUpdate(STATUS status, Object message) { @@ -616,16 +647,16 @@ public abstract class AbstractRipper } observer.update(this, new RipStatusMessage(status, message)); } - + /** * Get the completion percentage. - * @return - * Percentage complete + * + * @return Percentage complete */ public abstract int getCompletionPercentage(); + /** - * @return - * Text for status + * @return Text for status */ public abstract String getStatusText(); @@ -647,6 +678,7 @@ public abstract class AbstractRipper cleanup(); } } + /** * Tries to delete any empty directories */ @@ -656,18 +688,18 @@ public abstract class AbstractRipper LOGGER.info("Deleting empty directory " + this.workingDir); boolean deleteResult = this.workingDir.delete(); if (!deleteResult) { - LOGGER.error("Unable to delete empty directory " + this.workingDir); + LOGGER.error("Unable to delete empty directory " + this.workingDir); } } } - + /** * Pauses thread for a set amount of time. - * @param milliseconds - * Amount of time (in milliseconds) that the thread gets paused for - * @return - * True if paused successfully - * False if failed to pause/got interrupted. + * + * @param milliseconds Amount of time (in milliseconds) that the thread gets + * paused for + * @return True if paused successfully + * False if failed to pause/got interrupted. */ protected boolean sleep(int milliseconds) { try { @@ -683,6 +715,7 @@ public abstract class AbstractRipper public void setBytesTotal(int bytes) { // Do nothing } + public void setBytesCompleted(int bytes) { // Do nothing } @@ -692,20 +725,28 @@ public abstract class AbstractRipper LOGGER.debug("THIS IS A TEST RIP"); thisIsATest = true; } + protected static boolean isThisATest() { return thisIsATest; } // If true ripme uses a byte progress bar - protected boolean useByteProgessBar() { return false;} + protected boolean useByteProgessBar() { + return false; + } + // If true ripme will try to resume a broken download for this ripper - protected boolean tryResumeDownload() { return false;} + protected boolean tryResumeDownload() { + return false; + } protected boolean shouldIgnoreURL(URL url) { final String[] ignoredExtensions = Utils.getConfigStringArray("download.ignore_extensions"); - if (ignoredExtensions == null || ignoredExtensions.length == 0) return false; // nothing ignored + if (ignoredExtensions == null || ignoredExtensions.length == 0) + return false; // nothing ignored String[] pathElements = url.getPath().split("\\."); - if (pathElements.length == 0) return false; // no extension, can't filter + if (pathElements.length == 0) + return false; // no extension, can't filter String extension = pathElements[pathElements.length - 1]; for (String ignoredExtension : ignoredExtensions) { if (ignoredExtension.equalsIgnoreCase(extension)) { @@ -714,4 +755,4 @@ public abstract class AbstractRipper } return false; } -} \ No newline at end of file +}