From 9946777e812f76db2c6f55b0e33793e50fece4ec Mon Sep 17 00:00:00 2001 From: 4pr0n Date: Sat, 19 Apr 2014 22:41:11 -0700 Subject: [PATCH] Added video ripper support, xvideos for now #18 --- pom.xml | 2 +- .../ripme/ripper/AbstractRipper.java | 135 ++++------------ .../rarchives/ripme/ripper/AlbumRipper.java | 146 ++++++++++++++++++ .../ripme/ripper/DownloadVideoThread.java | 126 +++++++++++++++ .../rarchives/ripme/ripper/VideoRipper.java | 121 +++++++++++++++ .../ripme/ripper/rippers/ChanRipper.java | 4 +- .../ripper/rippers/DeviantartRipper.java | 4 +- .../ripper/rippers/EightmusesRipper.java | 4 +- .../ripme/ripper/rippers/GonewildRipper.java | 4 +- .../ripme/ripper/rippers/ImagearnRipper.java | 4 +- .../ripme/ripper/rippers/ImagefapRipper.java | 4 +- .../ripme/ripper/rippers/ImgurRipper.java | 4 +- .../ripme/ripper/rippers/InstagramRipper.java | 4 +- .../ripper/rippers/KinkyshareRipper.java | 4 +- .../ripper/rippers/MotherlessRipper.java | 4 +- .../ripme/ripper/rippers/RedditRipper.java | 6 +- .../ripme/ripper/rippers/SeeniveRipper.java | 4 +- .../ripme/ripper/rippers/TumblrRipper.java | 4 +- .../ripme/ripper/rippers/TwitterRipper.java | 4 +- .../ripme/ripper/rippers/VineboxRipper.java | 4 +- .../ripme/ripper/rippers/VkRipper.java | 4 +- .../ripme/ripper/rippers/XhamsterRipper.java | 4 +- .../ripper/rippers/video/XvideosRipper.java | 84 ++++++++++ .../com/rarchives/ripme/ui/MainWindow.java | 7 + .../rarchives/ripme/ui/RipStatusMessage.java | 4 +- .../com/rarchives/ripme/ui/UpdateUtils.java | 2 +- .../java/com/rarchives/ripme/utils/Utils.java | 11 ++ .../java/com/rarchives/ripme/tst/AppTest.java | 3 + .../tst/ripper/rippers/VideoRippersTest.java | 32 ++++ 29 files changed, 599 insertions(+), 144 deletions(-) create mode 100644 src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java create mode 100644 src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java create mode 100644 src/main/java/com/rarchives/ripme/ripper/VideoRipper.java create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java diff --git a/pom.xml b/pom.xml index 5ce82a49..9390b512 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rarchives.ripme ripme jar - 1.0.19 + 1.0.20 ripme http://rip.rarchives.com diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 1bc9f261..cff22079 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -6,10 +6,7 @@ import java.lang.reflect.Constructor; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Observable; import org.apache.log4j.Logger; @@ -23,7 +20,7 @@ public abstract class AbstractRipper extends Observable implements RipperInterface, Runnable { - private static final Logger logger = Logger.getLogger(AbstractRipper.class); + protected static final Logger logger = Logger.getLogger(AbstractRipper.class); protected static final String USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:27.0) Gecko/20100101 Firefox/27.0"; @@ -33,9 +30,6 @@ public abstract class AbstractRipper protected DownloadThreadPool threadPool; protected RipStatusHandler observer = null; - protected Map itemsPending = Collections.synchronizedMap(new HashMap()); - protected Map itemsCompleted = Collections.synchronizedMap(new HashMap()); - protected Map itemsErrored = Collections.synchronizedMap(new HashMap()); protected boolean completed = true; public abstract void rip() throws IOException; @@ -102,7 +96,7 @@ public abstract class AbstractRipper // Use empty subdirectory addURLToDownload(url, prefix, ""); } - + /** * Queues image to be downloaded and saved. * @param url @@ -110,17 +104,7 @@ public abstract class AbstractRipper * @param saveAs * Path of the local file to save the content to. */ - public void addURLToDownload(URL url, File saveAs) { - if (itemsPending.containsKey(url) - || itemsCompleted.containsKey(url) - || itemsErrored.containsKey(url)) { - // Item is already downloaded/downloading, skip it. - logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); - return; - } - itemsPending.put(url, saveAs); - threadPool.addThread(new DownloadFileThread(url, saveAs, this)); - } + public abstract void addURLToDownload(URL url, File saveAs); /** * Queues file to be downloaded and saved. With options. @@ -192,67 +176,30 @@ public abstract class AbstractRipper * @param saveAs * Where the downloaded file is stored. */ - public void downloadCompleted(URL url, File saveAs) { - if (observer == null) { - return; - } - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - itemsPending.remove(url); - itemsCompleted.put(url, saveAs); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - + public abstract void downloadCompleted(URL url, File saveAs); /** * Notifies observers that a file could not be downloaded (includes a reason). * @param url * @param reason */ - public void downloadErrored(URL url, String reason) { - if (observer == null) { - return; - } - itemsPending.remove(url); - itemsErrored.put(url, reason); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); - - checkIfComplete(); - } - + public abstract void downloadErrored(URL url, String reason); /** * Notify observers that a download could not be completed, * but was not technically an "error". * @param url * @param message */ - public void downloadProblem(URL url, String message) { - if (observer == null) { - return; - } - - itemsPending.remove(url); - itemsErrored.put(url, message); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message)); - - - checkIfComplete(); - } + public abstract void downloadProblem(URL url, String message); /** * Notifies observers and updates state if all files have been ripped. */ - private void checkIfComplete() { + protected void checkIfComplete() { if (observer == null) { return; } - if (!completed && itemsPending.isEmpty()) { + if (!completed) { completed = true; logger.info(" Rip completed!"); @@ -274,26 +221,7 @@ public abstract class AbstractRipper return workingDir; } - /** - * Sets directory to save all ripped files to. - * @param url - * URL to define how the workin directory should be saved. - */ - public void setWorkingDir(URL url) throws IOException { - String path = Utils.getWorkingDirectory().getCanonicalPath(); - if (!path.endsWith(File.separator)) { - path += File.separator; - } - String title = getAlbumTitle(this.url); - title = Utils.filesystemSafe(title); - path += title + File.separator; - this.workingDir = new File(path); - if (!this.workingDir.exists()) { - logger.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir)); - this.workingDir.mkdirs(); - } - logger.debug("Set working directory to: " + this.workingDir); - } + public abstract void setWorkingDir(URL url) throws IOException; public String getAlbumTitle(URL url) throws MalformedURLException { return getHost() + "_" + getGID(url); @@ -309,9 +237,17 @@ public abstract class AbstractRipper * If no compatible rippers can be found. */ public static AbstractRipper getRipper(URL url) throws Exception { - for (Constructor constructor : getRipperConstructors()) { + for (Constructor constructor : getRipperConstructors("com.rarchives.ripme.ripper.rippers")) { try { - AbstractRipper ripper = (AbstractRipper) constructor.newInstance(url); + AlbumRipper ripper = (AlbumRipper) constructor.newInstance(url); + return ripper; + } catch (Exception e) { + // Incompatible rippers *will* throw exceptions during instantiation. + } + } + for (Constructor constructor : getRipperConstructors("com.rarchives.ripme.ripper.rippers.video")) { + try { + VideoRipper ripper = (VideoRipper) constructor.newInstance(url); return ripper; } catch (Exception e) { // Incompatible rippers *will* throw exceptions during instantiation. @@ -325,9 +261,9 @@ public abstract class AbstractRipper * List of constructors for all eligible Rippers. * @throws Exception */ - private static List> getRipperConstructors() throws Exception { + private static List> getRipperConstructors(String pkg) throws Exception { List> constructors = new ArrayList>(); - for (Class clazz : Utils.getClassesForPackage("com.rarchives.ripme.ripper.rippers")) { + for (Class clazz : Utils.getClassesForPackage(pkg)) { if (AbstractRipper.class.isAssignableFrom(clazz)) { constructors.add( (Constructor) clazz.getConstructor(URL.class) ); } @@ -347,28 +283,9 @@ public abstract class AbstractRipper observer.update(this, new RipStatusMessage(status, message)); } - /** - * @return - * Integer between 0 and 100 defining the progress of the album rip. - */ - public int getCompletionPercentage() { - double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size(); - return (int) (100 * ( (total - itemsPending.size()) / total)); - } + public abstract int getCompletionPercentage(); - /** - * @return - * Human-readable information on the status of the current rip. - */ - public String getStatusText() { - StringBuilder sb = new StringBuilder(); - sb.append(getCompletionPercentage()) - .append("% ") - .append("- Pending: " ).append(itemsPending.size()) - .append(", Completed: ").append(itemsCompleted.size()) - .append(", Errored: " ).append(itemsErrored.size()); - return sb.toString(); - } + public abstract String getStatusText(); /** * Rips the album when the thread is invoked. @@ -382,4 +299,10 @@ public abstract class AbstractRipper } } + public void setBytesTotal(int bytes) { + // Do nothing + } + public void setBytesCompleted(int bytes) { + // Do nothing + } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java new file mode 100644 index 00000000..ca456737 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -0,0 +1,146 @@ +package com.rarchives.ripme.ripper; + +import java.io.File; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import com.rarchives.ripme.ui.RipStatusMessage; +import com.rarchives.ripme.ui.RipStatusMessage.STATUS; +import com.rarchives.ripme.utils.Utils; + +public abstract class AlbumRipper extends AbstractRipper { + + protected Map itemsPending = Collections.synchronizedMap(new HashMap()); + protected Map itemsCompleted = Collections.synchronizedMap(new HashMap()); + protected Map itemsErrored = Collections.synchronizedMap(new HashMap()); + + public AlbumRipper(URL url) throws IOException { + super(url); + } + + public abstract boolean canRip(URL url); + public abstract URL sanitizeURL(URL url) throws MalformedURLException; + public abstract void rip() throws IOException; + public abstract String getHost(); + public abstract String getGID(URL url) throws MalformedURLException; + + @Override + public void addURLToDownload(URL url, File saveAs) { + if (itemsPending.containsKey(url) + || itemsCompleted.containsKey(url) + || itemsErrored.containsKey(url)) { + // Item is already downloaded/downloading, skip it. + logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); + return; + } + itemsPending.put(url, saveAs); + threadPool.addThread(new DownloadFileThread(url, saveAs, this)); + } + + @Override + public void downloadCompleted(URL url, File saveAs) { + if (observer == null) { + return; + } + try { + String path = Utils.removeCWD(saveAs); + RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); + itemsPending.remove(url); + itemsCompleted.put(url, saveAs); + observer.update(this, msg); + + checkIfComplete(); + } catch (Exception e) { + logger.error("Exception while updating observer: ", e); + } + } + + @Override + public void downloadErrored(URL url, String reason) { + if (observer == null) { + return; + } + itemsPending.remove(url); + itemsErrored.put(url, reason); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); + + checkIfComplete(); + } + + @Override + public void downloadProblem(URL url, String message) { + if (observer == null) { + return; + } + + itemsPending.remove(url); + itemsErrored.put(url, message); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message)); + + checkIfComplete(); + } + + /** + * Notifies observers and updates state if all files have been ripped. + */ + @Override + protected void checkIfComplete() { + if (observer == null) { + return; + } + if (itemsPending.isEmpty()) { + super.checkIfComplete(); + } + } + + /** + * Sets directory to save all ripped files to. + * @param url + * URL to define how the workin directory should be saved. + */ + @Override + public void setWorkingDir(URL url) throws IOException { + String path = Utils.getWorkingDirectory().getCanonicalPath(); + if (!path.endsWith(File.separator)) { + path += File.separator; + } + String title = getAlbumTitle(this.url); + title = Utils.filesystemSafe(title); + path += title + File.separator; + this.workingDir = new File(path); + if (!this.workingDir.exists()) { + logger.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir)); + this.workingDir.mkdirs(); + } + logger.debug("Set working directory to: " + this.workingDir); + } + + /** + * @return + * Integer between 0 and 100 defining the progress of the album rip. + */ + @Override + public int getCompletionPercentage() { + double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size(); + return (int) (100 * ( (total - itemsPending.size()) / total)); + } + + /** + * @return + * Human-readable information on the status of the current rip. + */ + @Override + public String getStatusText() { + StringBuilder sb = new StringBuilder(); + sb.append(getCompletionPercentage()) + .append("% ") + .append("- Pending: " ).append(itemsPending.size()) + .append(", Completed: ").append(itemsCompleted.size()) + .append(", Errored: " ).append(itemsErrored.size()); + return sb.toString(); + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java new file mode 100644 index 00000000..d50cf31b --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -0,0 +1,126 @@ +package com.rarchives.ripme.ripper; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; + +import org.apache.log4j.Logger; + +import com.rarchives.ripme.ui.RipStatusMessage.STATUS; +import com.rarchives.ripme.utils.Utils; + +/** + * Thread for downloading files. + * Includes retry logic, observer notifications, and other goodies. + */ +public class DownloadVideoThread extends Thread { + + private static final Logger logger = Logger.getLogger(DownloadVideoThread.class); + + private URL url; + private File saveAs; + private String prettySaveAs; + private AbstractRipper observer; + private int retries; + + public DownloadVideoThread(URL url, File saveAs, AbstractRipper observer) { + super(); + this.url = url; + this.saveAs = saveAs; + this.prettySaveAs = Utils.removeCWD(saveAs); + this.observer = observer; + this.retries = Utils.getConfigInteger("download.retries", 1); + } + + /** + * Attempts to download the file. Retries as needed. + * Notifies observers upon completion/error/warn. + */ + public void run() { + try { + observer.stopCheck(); + } catch (IOException e) { + observer.downloadErrored(url, "Download interrupted"); + return; + } + if (saveAs.exists()) { + if (Utils.getConfigBoolean("file.overwrite", false)) { + logger.info("[!] Deleting existing file" + prettySaveAs); + saveAs.delete(); + } else { + logger.info("[!] Skipping " + url + " -- file already exists: " + prettySaveAs); + observer.downloadProblem(url, "File already exists: " + prettySaveAs); + return; + } + } + + int bytesTotal, bytesDownloaded = 0; + try { + bytesTotal = getTotalBytes(this.url); + } catch (IOException e) { + logger.error("Failed to get file size at " + this.url, e); + observer.downloadErrored(this.url, "Failed to get file size of " + this.url); + return; + } + observer.setBytesTotal(bytesTotal); + observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); + logger.info("Size of file at " + this.url + " = " + bytesTotal + "b"); + + int tries = 0; // Number of attempts to download + do { + InputStream bis = null; OutputStream fos = null; + byte[] data = new byte[1024]; int bytesRead; + try { + logger.info(" Downloading file: " + url + (tries > 0 ? " Retry #" + tries : "")); + observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); + tries += 1; + bis = new BufferedInputStream(this.url.openStream()); + fos = new FileOutputStream(saveAs); + while ( (bytesRead = bis.read(data)) != -1) { + try { + observer.stopCheck(); + } catch (IOException e) { + observer.downloadErrored(url, "Download interrupted"); + return; + } + fos.write(data, 0, bytesRead); + bytesDownloaded += bytesRead; + observer.setBytesCompleted(bytesDownloaded); + observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); + } + bis.close(); + fos.close(); + break; // Download successful: break out of infinite loop + } catch (IOException e) { + logger.error("[!] Exception while downloading file: " + url + " - " + e.getMessage(), e); + } finally { + // Close any open streams + try { + if (bis != null) { bis.close(); } + } catch (IOException e) { } + try { + if (fos != null) { fos.close(); } + } catch (IOException e) { } + } + if (tries > this.retries) { + logger.error("[!] Exceeded maximum retries (" + this.retries + ") for URL " + url); + observer.downloadErrored(url, "Failed to download " + url.toExternalForm()); + return; + } + } while (true); + observer.downloadCompleted(url, saveAs); + logger.info("[+] Saved " + url + " as " + this.prettySaveAs); + } + + private int getTotalBytes(URL url) throws IOException { + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("HEAD"); + return conn.getContentLength(); + } + +} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java new file mode 100644 index 00000000..ae52e0c3 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -0,0 +1,121 @@ +package com.rarchives.ripme.ripper; + +import java.io.File; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; + +import com.rarchives.ripme.ui.RipStatusMessage; +import com.rarchives.ripme.ui.RipStatusMessage.STATUS; +import com.rarchives.ripme.utils.Utils; + +public abstract class VideoRipper extends AbstractRipper { + + private int bytesTotal = 1, + bytesCompleted = 1; + + public VideoRipper(URL url) throws IOException { + super(url); + } + + public abstract boolean canRip(URL url); + public abstract void rip() throws IOException; + public abstract String getHost(); + public abstract String getGID(URL url) throws MalformedURLException; + + @Override + public void setBytesTotal(int bytes) { + this.bytesTotal = bytes; + } + @Override + public void setBytesCompleted(int bytes) { + this.bytesCompleted = bytes; + } + + @Override + public void addURLToDownload(URL url, File saveAs) { + threadPool.addThread(new DownloadVideoThread(url, saveAs, this)); + } + + @Override + public void setWorkingDir(URL url) throws IOException { + String path = Utils.getWorkingDirectory().getCanonicalPath(); + if (!path.endsWith(File.separator)) { + path += File.separator; + } + path += "videos" + File.separator; + this.workingDir = new File(path); + if (!this.workingDir.exists()) { + logger.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir)); + this.workingDir.mkdirs(); + } + logger.debug("Set working directory to: " + this.workingDir); + } + + @Override + public int getCompletionPercentage() { + return (int) (100 * (bytesCompleted / (float) bytesTotal)); + } + + @Override + public void downloadCompleted(URL url, File saveAs) { + if (observer == null) { + return; + } + try { + String path = Utils.removeCWD(saveAs); + RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); + observer.update(this, msg); + + checkIfComplete(); + } catch (Exception e) { + logger.error("Exception while updating observer: ", e); + } + } + @Override + public void downloadErrored(URL url, String reason) { + if (observer == null) { + return; + } + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); + checkIfComplete(); + } + @Override + public void downloadProblem(URL url, String message) { + if (observer == null) { + return; + } + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message)); + checkIfComplete(); + } + + @Override + public String getStatusText() { + StringBuilder sb = new StringBuilder(); + sb.append(getCompletionPercentage()) + .append("% ") + .append(" - ") + .append(Utils.bytesToHumanReadable(bytesCompleted)) + .append(" / ") + .append(Utils.bytesToHumanReadable(bytesTotal)); + return sb.toString(); + } + + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + return url; + } + + /** + * Notifies observers and updates state if all files have been ripped. + */ + @Override + protected void checkIfComplete() { + if (observer == null) { + return; + } + if (bytesCompleted >= bytesTotal) { + super.checkIfComplete(); + } + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java index 71bf73fe..8f10a3ff 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java @@ -13,9 +13,9 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.AlbumRipper; -public class ChanRipper extends AbstractRipper { +public class ChanRipper extends AlbumRipper { private static final Logger logger = Logger.getLogger(ChanRipper.class); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index f37b347e..d0fc3f98 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -18,10 +18,10 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.AlbumRipper; import com.rarchives.ripme.utils.Utils; -public class DeviantartRipper extends AbstractRipper { +public class DeviantartRipper extends AlbumRipper { private static final String DOMAIN = "deviantart.com", HOST = "deviantart"; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java index 9d0559bb..b0ed61b2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java @@ -12,9 +12,9 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.AlbumRipper; -public class EightmusesRipper extends AbstractRipper { +public class EightmusesRipper extends AlbumRipper { private static final String DOMAIN = "8muses.com", HOST = "8muses"; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GonewildRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GonewildRipper.java index aac913d2..40a376ca 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GonewildRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GonewildRipper.java @@ -11,10 +11,10 @@ import org.json.JSONArray; import org.json.JSONObject; import org.jsoup.Jsoup; -import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.AlbumRipper; import com.rarchives.ripme.utils.Utils; -public class GonewildRipper extends AbstractRipper { +public class GonewildRipper extends AlbumRipper { private static final String HOST = "gonewild"; private static final Logger logger = Logger.getLogger(GonewildRipper.class); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java index 515e9443..c08b7167 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java @@ -11,9 +11,9 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.AlbumRipper; -public class ImagearnRipper extends AbstractRipper { +public class ImagearnRipper extends AlbumRipper { private static final String DOMAIN = "imagearn.com", HOST = "imagearn"; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java index a69f8748..a37a0017 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java @@ -11,9 +11,9 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.AlbumRipper; -public class ImagefapRipper extends AbstractRipper { +public class ImagefapRipper extends AlbumRipper { private static final String DOMAIN = "imagefap.com", HOST = "imagefap"; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index b8ab02a9..2d9470c2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -18,11 +18,11 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; -import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.AlbumRipper; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; -public class ImgurRipper extends AbstractRipper { +public class ImgurRipper extends AlbumRipper { private static final String DOMAIN = "imgur.com", HOST = "imgur"; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index 718c57d7..3f39fd4f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -13,9 +13,9 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.AlbumRipper; -public class InstagramRipper extends AbstractRipper { +public class InstagramRipper extends AlbumRipper { private static final String DOMAIN = "instagram.com", HOST = "instagram"; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/KinkyshareRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/KinkyshareRipper.java index 702fc537..dade912d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/KinkyshareRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/KinkyshareRipper.java @@ -11,9 +11,9 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.AlbumRipper; -public class KinkyshareRipper extends AbstractRipper { +public class KinkyshareRipper extends AlbumRipper { private static final String HOST = "kinkyshare"; private static final Logger logger = Logger.getLogger(KinkyshareRipper.class); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java index daf33c39..52bfb33f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java @@ -11,10 +11,10 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.AlbumRipper; import com.rarchives.ripme.ripper.DownloadThreadPool; -public class MotherlessRipper extends AbstractRipper { +public class MotherlessRipper extends AlbumRipper { private static final String DOMAIN = "motherless.com", HOST = "motherless"; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index a6c03458..e12d894d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.SocketTimeoutException; import java.net.URL; import java.util.List; import java.util.regex.Matcher; @@ -14,12 +15,11 @@ import org.json.JSONTokener; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; -import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.AlbumRipper; import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; -import java.net.SocketTimeoutException; -public class RedditRipper extends AbstractRipper { +public class RedditRipper extends AlbumRipper { public RedditRipper(URL url) throws IOException { super(url); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java index 4ec4b649..86c815f1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java @@ -12,10 +12,10 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.AlbumRipper; import com.rarchives.ripme.ripper.DownloadThreadPool; -public class SeeniveRipper extends AbstractRipper { +public class SeeniveRipper extends AlbumRipper { private static final String DOMAIN = "seenive.com", HOST = "seenive"; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java index ee924f44..21f23bdf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java @@ -12,10 +12,10 @@ import org.json.JSONObject; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; -import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.AlbumRipper; import com.rarchives.ripme.utils.Utils; -public class TumblrRipper extends AbstractRipper { +public class TumblrRipper extends AlbumRipper { private static final String DOMAIN = "tumblr.com", HOST = "tumblr"; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java index 810734cb..e5cae727 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java @@ -16,10 +16,10 @@ import org.json.JSONTokener; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; -import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.AlbumRipper; import com.rarchives.ripme.utils.Utils; -public class TwitterRipper extends AbstractRipper { +public class TwitterRipper extends AlbumRipper { private static final String DOMAIN = "twitter.com", HOST = "twitter"; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java index 96fadcf7..8ddb2e3c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java @@ -12,9 +12,9 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.AlbumRipper; -public class VineboxRipper extends AbstractRipper { +public class VineboxRipper extends AlbumRipper { private static final String DOMAIN = "vinebox.co", HOST = "vinebox"; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java index efcadaf7..a3654e9f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java @@ -18,9 +18,9 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.AlbumRipper; -public class VkRipper extends AbstractRipper { +public class VkRipper extends AlbumRipper { private static final String DOMAIN = "vk.com", HOST = "vk"; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index e692e5a2..5f05ee64 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -11,9 +11,9 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.ripper.AlbumRipper; -public class XhamsterRipper extends AbstractRipper { +public class XhamsterRipper extends AlbumRipper { private static final String DOMAIN = "xhamster.com", HOST = "xhamster"; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java new file mode 100644 index 00000000..468c6691 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java @@ -0,0 +1,84 @@ +package com.rarchives.ripme.ripper.rippers.video; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLDecoder; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.log4j.Logger; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import com.rarchives.ripme.ripper.VideoRipper; + +public class XvideosRipper extends VideoRipper { + + private static final String HOST = "xvideos"; + private static final Logger logger = Logger.getLogger(XvideosRipper.class); + + public XvideosRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return HOST; + } + + @Override + public boolean canRip(URL url) { + Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/video[0-9]+.*$"); + Matcher m = p.matcher(url.toExternalForm()); + return m.matches(); + } + + @Override + public String getAlbumTitle(URL url) { + return "videos"; + } + + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + return url; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/video([0-9]+).*$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + + throw new MalformedURLException( + "Expected xvideo format:" + + "xvideos.com/video####" + + " Got: " + url); + } + + @Override + public void rip() throws IOException { + logger.info(" Retrieving " + this.url.toExternalForm()); + Document doc = Jsoup.connect(this.url.toExternalForm()) + .userAgent(USER_AGENT) + .get(); + Elements embeds = doc.select("embed"); + if (embeds.size() == 0) { + throw new IOException("Could not find Embed code at " + url); + } + Element embed = embeds.get(0); + String vars = embed.attr("flashvars"); + for (String var : vars.split("&")) { + if (var.startsWith("flv_url=")) { + String vidUrl = var.substring("flv_url=".length()); + vidUrl = URLDecoder.decode(vidUrl, "UTF-8"); + addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url)); + } + } + waitForThreads(); + } +} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index d83f07ad..ad2f0550 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -721,6 +721,13 @@ public class MainWindow implements Runnable, RipStatusHandler { } }); mainFrame.pack(); + break; + case COMPLETED_BYTES: + // Update completed bytes + break; + case TOTAL_BYTES: + // Update total bytes + break; } } diff --git a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java index 45c01edc..fee3fc59 100644 --- a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java +++ b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java @@ -11,7 +11,9 @@ public class RipStatusMessage { DOWNLOAD_COMPLETE("Download Complete"), DOWNLOAD_ERRORED("Download Errored"), RIP_COMPLETE("Rip Complete"), - DOWNLOAD_WARN("Download problem"); + DOWNLOAD_WARN("Download problem"), + TOTAL_BYTES("Total bytes"), + COMPLETED_BYTES("Completed bytes"); String value; STATUS(String value) { diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 3bdb62eb..f1d24ad8 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -19,7 +19,7 @@ import org.jsoup.nodes.Document; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.0.19"; + private static final String DEFAULT_VERSION = "1.0.20"; private static final String updateJsonURL = "http://rarchives.com/ripme.json"; private static final String updateJarURL = "http://rarchives.com/ripme.jar"; private static final String mainFileName = "ripme.jar"; diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 8d20837d..2500984a 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -247,4 +247,15 @@ public class Utils { .replaceAll("__", "_") .replaceAll("_+$", ""); } + + public static String bytesToHumanReadable(int bytes) { + float fbytes = (float) bytes; + String[] mags = new String[] {"", "k", "m", "g", "t"}; + int magIndex = 0; + while (fbytes >= 1024) { + fbytes /= 1024; + magIndex++; + } + return String.format("%.2f%sb", fbytes, mags[magIndex]); + } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/AppTest.java b/src/test/java/com/rarchives/ripme/tst/AppTest.java index 630e46d9..77368a89 100644 --- a/src/test/java/com/rarchives/ripme/tst/AppTest.java +++ b/src/test/java/com/rarchives/ripme/tst/AppTest.java @@ -4,6 +4,8 @@ import junit.framework.Test; import junit.framework.TestCase; import junit.framework.TestSuite; +import com.rarchives.ripme.utils.Utils; + public class AppTest extends TestCase { /** * Create the test case @@ -25,6 +27,7 @@ public class AppTest extends TestCase { * Rigourous Test :-) */ public void testApp() { + System.err.println(Utils.bytesToHumanReadable(1023 * 5000)); assertTrue( true ); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java new file mode 100644 index 00000000..847700a9 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java @@ -0,0 +1,32 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +import com.rarchives.ripme.ripper.rippers.video.XvideosRipper; + +public class VideoRippersTest extends RippersTest { + + public void testXvideosRipper() throws IOException { + if (false && !DOWNLOAD_CONTENT) { + return; + } + List contentURLs = new ArrayList(); + contentURLs.add(new URL("http://www.xvideos.com/video1428195/stephanie_first_time_anal")); + contentURLs.add(new URL("http://www.xvideos.com/video7136868/vid-20140205-wa0011")); + for (URL url : contentURLs) { + try { + XvideosRipper ripper = new XvideosRipper(url); + ripper.rip(); + assert(ripper.getWorkingDir().listFiles().length > 1); + deleteDir(ripper.getWorkingDir()); + } catch (Exception e) { + e.printStackTrace(); + fail("Error while ripping URL " + url + ": " + e.getMessage()); + } + } + } + +}