From 8b6c66da0c6730355eb4c679fbe8ae963e8158ca Mon Sep 17 00:00:00 2001 From: Erwin de Haan Date: Wed, 15 Oct 2014 14:02:36 +0200 Subject: [PATCH] Fixed the Finebox ripper --- nbactions.xml | 17 +++++++++ .../ripme/ripper/AbstractRipper.java | 20 +++++----- .../rarchives/ripme/ripper/AlbumRipper.java | 17 ++++++--- ...{VineboxRipper.java => FineboxRipper.java} | 37 ++++++++++++------- ...RipperTest.java => FineboxRipperTest.java} | 7 ++-- 5 files changed, 66 insertions(+), 32 deletions(-) create mode 100644 nbactions.xml rename src/main/java/com/rarchives/ripme/ripper/rippers/{VineboxRipper.java => FineboxRipper.java} (60%) rename src/test/java/com/rarchives/ripme/tst/ripper/rippers/{VineboxRipperTest.java => FineboxRipperTest.java} (76%) diff --git a/nbactions.xml b/nbactions.xml new file mode 100644 index 00000000..fa676bd9 --- /dev/null +++ b/nbactions.xml @@ -0,0 +1,17 @@ + + + + run + + jar + + + process-classes + org.codehaus.mojo:exec-maven-plugin:1.2.1:exec + + + -classpath %classpath com.rarchives.ripme.App + java + + + diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index f674e5cc..e1ea4a23 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -94,14 +94,14 @@ public abstract class AbstractRipper * @param saveAs * Path of the local file to save the content to. */ - public abstract void addURLToDownload(URL url, File saveAs); - public abstract void addURLToDownload(URL url, File saveAs, String referrer, Map cookies); + public abstract boolean addURLToDownload(URL url, File saveAs); + public abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies); - public void addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies) { + public boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies) { try { stopCheck(); } catch (IOException e) { - return; + return false; } String saveAs = url.toExternalForm(); saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1); @@ -122,14 +122,14 @@ public abstract class AbstractRipper + saveAs); } catch (IOException e) { logger.error("[!] Error creating save file path for URL '" + url + "':", e); - return; + return false; } logger.debug("Downloading " + url + " to " + saveFileAs); if (!saveFileAs.getParentFile().exists()) { logger.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); saveFileAs.getParentFile().mkdirs(); } - addURLToDownload(url, saveFileAs, referrer, cookies); + return addURLToDownload(url, saveFileAs, referrer, cookies); } /** @@ -141,8 +141,8 @@ public abstract class AbstractRipper * @param subdirectory * Sub-directory of the working directory to save the images to. */ - public void addURLToDownload(URL url, String prefix, String subdirectory) { - addURLToDownload(url, prefix, subdirectory, null, null); + public boolean addURLToDownload(URL url, String prefix, String subdirectory) { + return addURLToDownload(url, prefix, subdirectory, null, null); } /** @@ -153,9 +153,9 @@ public abstract class AbstractRipper * @param prefix * Text to append to saved filename. */ - public void addURLToDownload(URL url, String prefix) { + public boolean addURLToDownload(URL url, String prefix) { // Use empty subdirectory - addURLToDownload(url, prefix, ""); + return addURLToDownload(url, prefix, ""); } /** * Waits for downloading threads to complete. diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 09f117af..b4135b22 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -33,14 +33,14 @@ public abstract class AlbumRipper extends AbstractRipper { return false; } - public void addURLToDownload(URL url, File saveAs, String referrer, Map cookies) { + public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies) { if (!allowDuplicates() && ( itemsPending.containsKey(url) || itemsCompleted.containsKey(url) || itemsErrored.containsKey(url) )) { // Item is already downloaded/downloading, skip it. logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); - return; + return false; } if (Utils.getConfigBoolean("urls_only.save", false)) { // Output URL to file @@ -68,11 +68,12 @@ public abstract class AlbumRipper extends AbstractRipper { } threadPool.addThread(dft); } + return true; } @Override - public void addURLToDownload(URL url, File saveAs) { - addURLToDownload(url, saveAs, null, null); + public boolean addURLToDownload(URL url, File saveAs) { + return addURLToDownload(url, saveAs, null, null); } /** @@ -80,10 +81,12 @@ public abstract class AlbumRipper extends AbstractRipper { * Uses filename from URL to decide filename. * @param url * URL to download + * @return + * True on success */ - public void addURLToDownload(URL url) { + public boolean addURLToDownload(URL url) { // Use empty prefix and empty subdirectory - addURLToDownload(url, "", ""); + return addURLToDownload(url, "", ""); } @Override @@ -146,6 +149,8 @@ public abstract class AlbumRipper extends AbstractRipper { * Sets directory to save all ripped files to. * @param url * URL to define how the working directory should be saved. + * @throws + * IOException */ @Override public void setWorkingDir(URL url) throws IOException { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FineboxRipper.java similarity index 60% rename from src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java rename to src/main/java/com/rarchives/ripme/ripper/rippers/FineboxRipper.java index e04cac14..dc1393f6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FineboxRipper.java @@ -13,31 +13,34 @@ import org.jsoup.nodes.Element; import com.rarchives.ripme.ripper.AlbumRipper; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; +import org.jsoup.select.Elements; -public class VineboxRipper extends AlbumRipper { +public class FineboxRipper extends AlbumRipper { - private static final String DOMAIN = "vinebox.co", - HOST = "vinebox"; + private static final String DOMAIN = "finebox.co", + DOMAIN_OLD = "vinebox.co", + HOST = "finebox"; - public VineboxRipper(URL url) throws IOException { + public FineboxRipper(URL url) throws IOException { super(url); } @Override public boolean canRip(URL url) { - return url.getHost().endsWith(DOMAIN); + return url.getHost().endsWith(DOMAIN) || url.getHost().endsWith(DOMAIN_OLD); } @Override public URL sanitizeURL(URL url) throws MalformedURLException { - return new URL("http://vinebox.co/u/" + getGID(url)); + return new URL("http://"+DOMAIN+"/u/" + getGID(url)); } @Override public void rip() throws IOException { int page = 0; Document doc; - while (true) { + Boolean hasPagesLeft = true; + while (hasPagesLeft) { page++; String urlPaged = this.url.toExternalForm() + "?page=" + page; logger.info("Retrieving " + urlPaged); @@ -47,10 +50,18 @@ public class VineboxRipper extends AlbumRipper { } catch (HttpStatusException e) { logger.debug("Hit end of pages at page " + page, e); break; - } - for (Element element : doc.select("video")) { - addURLToDownload(new URL(element.attr("src"))); - } + } + Elements videos = doc.select("video"); + for (Element element : videos) { + String videourl = element.attr("src"); + if(videourl.substring(0,4)!="http"){ + videourl = "http://"+DOMAIN+ videourl; + } + if(!addURLToDownload(new URL(videourl))){ + hasPagesLeft = false; + break; + } + } try { Thread.sleep(1000); } catch (InterruptedException e) { @@ -68,10 +79,10 @@ public class VineboxRipper extends AlbumRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://(www\\.)?vinebox\\.co/u/([a-zA-Z0-9]{1,}).*$"); + Pattern p = Pattern.compile("^https?://(www\\.)?(v|f)inebox\\.co/u/([a-zA-Z0-9]{1,}).*$"); Matcher m = p.matcher(url.toExternalForm()); if (!m.matches()) { - throw new MalformedURLException("Expected format: http://vinebox.co/u/USERNAME"); + throw new MalformedURLException("Expected format: http://"+DOMAIN+"/u/USERNAME"); } return m.group(m.groupCount()); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineboxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FineboxRipperTest.java similarity index 76% rename from src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineboxRipperTest.java rename to src/test/java/com/rarchives/ripme/tst/ripper/rippers/FineboxRipperTest.java index 734d4cbd..92cc4f24 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VineboxRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FineboxRipperTest.java @@ -5,9 +5,9 @@ import java.net.URL; import java.util.ArrayList; import java.util.List; -import com.rarchives.ripme.ripper.rippers.VineboxRipper; +import com.rarchives.ripme.ripper.rippers.FineboxRipper; -public class VineboxRipperTest extends RippersTest { +public class FineboxRipperTest extends RippersTest { public void testVineboxAlbums() throws IOException { if (DOWNLOAD_CONTENT) { @@ -15,9 +15,10 @@ public class VineboxRipperTest extends RippersTest { } List contentURLs = new ArrayList(); contentURLs.add(new URL("http://vinebox.co/u/wi57hMjc2Ka")); + contentURLs.add(new URL("http://finebox.co/u/wi57hMjc2Ka")); for (URL url : contentURLs) { try { - VineboxRipper ripper = new VineboxRipper(url); + FineboxRipper ripper = new FineboxRipper(url); ripper.rip(); assert(ripper.getWorkingDir().listFiles().length > 1); deleteDir(ripper.getWorkingDir());