From debba69c7f3cf03ecdd4c1b486e8b223614866c2 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 28 Oct 2018 07:54:49 -0400 Subject: [PATCH 1/5] Fixed error where on 403 reddit ripper would try to download a null url --- .../com/rarchives/ripme/ripper/rippers/RedditRipper.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 20d31d78..172250e3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -249,7 +249,10 @@ public class RedditRipper extends AlbumRipper { if (url.contains("v.redd.it")) { String savePath = this.workingDir + File.separator; savePath += id + "-" + url.split("/")[3] + title + ".mp4"; - addURLToDownload(parseRedditVideoMPD(urls.get(0).toExternalForm()), new File(savePath)); + URL urlToDownload = parseRedditVideoMPD(urls.get(0).toExternalForm()); + if (urlToDownload != null) { + addURLToDownload(urlToDownload, new File(savePath)); + } } else { addURLToDownload(urls.get(0), id + title, "", theUrl, null); From 5b78ba0266e3a7a415c28028d522b3ebe3797650 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 28 Oct 2018 08:07:13 -0400 Subject: [PATCH 2/5] DownloadFileThread now removes all illegal chars from filenames before trying to write the file to disk --- .../com/rarchives/ripme/ripper/DownloadFileThread.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 77a59a3e..54619a28 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -62,12 +62,19 @@ class DownloadFileThread extends Thread { this.cookies = cookies; } + public File sanitizeSaveAs(File fileToSan) { + String fileName = fileToSan.getName().replaceAll("[\\\\/:*?\"<>|]", "_"); + return new File(saveAs.getParentFile().getAbsolutePath() + File.separator + fileName); + } + /** * Attempts to download the file. Retries as needed. * Notifies observers upon completion/error/warn. */ public void run() { + // First thing we make sure the file name doesn't have any illegal chars in it + saveAs = sanitizeSaveAs(saveAs); long fileSize = 0; int bytesTotal = 0; int bytesDownloaded = 0; From 3929813c5a44ca2ddd5ea1718f94956471ac12fc Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 28 Oct 2018 09:06:15 -0400 Subject: [PATCH 3/5] Changed up name sanitization a bit --- .../ripme/ripper/DownloadFileThread.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 54619a28..639902a0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -20,7 +20,7 @@ import org.jsoup.HttpStatusException; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; -import com.rarchives.ripme.ripper.AbstractRipper; +import static java.lang.Math.toIntExact; /** * Thread for downloading files. @@ -62,11 +62,6 @@ class DownloadFileThread extends Thread { this.cookies = cookies; } - public File sanitizeSaveAs(File fileToSan) { - String fileName = fileToSan.getName().replaceAll("[\\\\/:*?\"<>|]", "_"); - return new File(saveAs.getParentFile().getAbsolutePath() + File.separator + fileName); - } - /** * Attempts to download the file. Retries as needed. @@ -74,7 +69,7 @@ class DownloadFileThread extends Thread { */ public void run() { // First thing we make sure the file name doesn't have any illegal chars in it - saveAs = sanitizeSaveAs(saveAs); + saveAs = new File(saveAs.getParentFile().getAbsolutePath() + File.separator + Utils.sanitizeSaveAs(saveAs.getName())); long fileSize = 0; int bytesTotal = 0; int bytesDownloaded = 0; @@ -222,14 +217,16 @@ class DownloadFileThread extends Thread { String[] saveAsSplit = saveAs.getName().split("\\."); // Get the file extension so when we shorten the file name we don't cut off the file extension String fileExt = saveAsSplit[saveAsSplit.length - 1]; - // The max limit for filenames on Linux with Ext3/4 is 255 bytes, on windows it's 256 chars so rather than - // bother with code with both platforms we just cut the file name down to 254 chars + // The max limit for filenames on Linux with Ext3/4 is 255 bytes logger.info(saveAs.getName().substring(0, 254 - fileExt.length()) + fileExt); String filename = saveAs.getName().substring(0, 254 - fileExt.length()) + "." + fileExt; // We can't just use the new file name as the saveAs because the file name doesn't include the // users save path, so we get the user save path from the old saveAs - saveAs = new File(saveAs.getParentFile().getAbsolutePath() + "/" + filename); + saveAs = new File(saveAs.getParentFile().getAbsolutePath() + File.separator + filename); fos = new FileOutputStream(saveAs); + } else if (saveAs.getAbsolutePath().length() > 259 && Utils.isWindows()) { + // This if is for when the file path has gone above 260 chars which windows does not allow + fos = new FileOutputStream(Utils.shortenSaveAsWindows(saveAs.getParentFile().getPath(), saveAs.getName())); } } } From d810f9c831d71335bfa1cb497753a97a0559b9e9 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 28 Oct 2018 09:06:50 -0400 Subject: [PATCH 4/5] Moved sanitizeSaveAs and shortenSaveAsWindows to Utils.java --- .../java/com/rarchives/ripme/utils/Utils.java | 40 ++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 1f49b8a5..32a5c70c 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -11,10 +11,7 @@ import javax.sound.sampled.AudioSystem; import javax.sound.sampled.Clip; import javax.sound.sampled.Line; import javax.sound.sampled.LineEvent; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.UnsupportedEncodingException; +import java.io.*; import java.lang.reflect.Array; import java.lang.reflect.Constructor; import java.net.URISyntaxException; @@ -33,6 +30,8 @@ import java.util.ResourceBundle; import java.util.jar.JarEntry; import java.util.jar.JarFile; +import static java.lang.Math.toIntExact; + /** * Common utility functions used in various places throughout the project. */ @@ -179,7 +178,7 @@ public class Utils { /** * Determines if your current system is a Windows system. */ - private static boolean isWindows() { + public static boolean isWindows() { return OS.contains("win"); } @@ -773,4 +772,35 @@ public class Utils { return false; } + public static String sanitizeSaveAs(String fileNameToSan) { + return fileNameToSan.replaceAll("[\\\\/:*?\"<>|]", "_"); + } + + public static File shortenSaveAsWindows(String ripsDirPath, String fileName) throws FileNotFoundException { +// int ripDirLength = ripsDirPath.length(); +// int maxFileNameLength = 260 - ripDirLength; +// LOGGER.info(maxFileNameLength); + LOGGER.error("The filename " + fileName + " is to long to be saved on this file system."); + LOGGER.info("Shortening filename"); + String fullPath = ripsDirPath + File.separator + fileName; + // How long the path without the file name is + int pathLength = ripsDirPath.length(); + int fileNameLength = fileName.length(); + LOGGER.info(pathLength); + LOGGER.info(fileNameLength); + if (pathLength == 260) { + // We've reached the max length, there's nothing more we can do + throw new FileNotFoundException("File path is too long for this OS"); + } + String[] saveAsSplit = fileName.split("\\."); + // Get the file extension so when we shorten the file name we don't cut off the file extension + String fileExt = saveAsSplit[saveAsSplit.length - 1]; + // The max limit for paths on Windows is 260 chars + LOGGER.info(fullPath.substring(0, 260 - pathLength - fileExt.length() + 1) + "." + fileExt); + fullPath = fullPath.substring(0, 260 - pathLength - fileExt.length() + 1) + "." + fileExt; + LOGGER.info(fullPath); + LOGGER.info(fullPath.length()); + return new File(fullPath); + } + } \ No newline at end of file From 81ca088dec12ae6b2e51a6c4051cded718794507 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 28 Oct 2018 09:07:10 -0400 Subject: [PATCH 5/5] Added unit tests for sanitizeSaveAs and shortenSaveAsWindows --- .../java/com/rarchives/ripme/tst/UtilsTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/UtilsTest.java b/src/test/java/com/rarchives/ripme/tst/UtilsTest.java index 3ffdd7b2..f51a617d 100644 --- a/src/test/java/com/rarchives/ripme/tst/UtilsTest.java +++ b/src/test/java/com/rarchives/ripme/tst/UtilsTest.java @@ -3,6 +3,8 @@ package com.rarchives.ripme.tst; import junit.framework.TestCase; import com.rarchives.ripme.utils.Utils; +import java.io.File; +import java.io.FileNotFoundException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -44,4 +46,15 @@ public class UtilsTest extends TestCase { assertEquals(Arrays.asList(" is a "), Utils.between("This is a test", "This", "test")); } + public void testShortenFileNameWindows() throws FileNotFoundException { + String filename = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff.png"; + // Test filename shortening for windows + File f = Utils.shortenSaveAsWindows("D:/rips/test/reddit/deep", filename ); + assertEquals(new File("D:/rips/test/reddit/deep/fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff.png"), f); + } + + public void testSanitizeSaveAs() { + assertEquals("This is a _ !__ test", Utils.sanitizeSaveAs("This is a \" !