diff --git a/pom.xml b/pom.xml
index 5ce82a49..9390b512 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
com.rarchives.ripme
ripme
jar
- 1.0.19
+ 1.0.20
ripme
http://rip.rarchives.com
diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
index 1bc9f261..cff22079 100644
--- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
@@ -6,10 +6,7 @@ import java.lang.reflect.Constructor;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.Observable;
import org.apache.log4j.Logger;
@@ -23,7 +20,7 @@ public abstract class AbstractRipper
extends Observable
implements RipperInterface, Runnable {
- private static final Logger logger = Logger.getLogger(AbstractRipper.class);
+ protected static final Logger logger = Logger.getLogger(AbstractRipper.class);
protected static final String USER_AGENT =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:27.0) Gecko/20100101 Firefox/27.0";
@@ -33,9 +30,6 @@ public abstract class AbstractRipper
protected DownloadThreadPool threadPool;
protected RipStatusHandler observer = null;
- protected Map itemsPending = Collections.synchronizedMap(new HashMap());
- protected Map itemsCompleted = Collections.synchronizedMap(new HashMap());
- protected Map itemsErrored = Collections.synchronizedMap(new HashMap());
protected boolean completed = true;
public abstract void rip() throws IOException;
@@ -102,7 +96,7 @@ public abstract class AbstractRipper
// Use empty subdirectory
addURLToDownload(url, prefix, "");
}
-
+
/**
* Queues image to be downloaded and saved.
* @param url
@@ -110,17 +104,7 @@ public abstract class AbstractRipper
* @param saveAs
* Path of the local file to save the content to.
*/
- public void addURLToDownload(URL url, File saveAs) {
- if (itemsPending.containsKey(url)
- || itemsCompleted.containsKey(url)
- || itemsErrored.containsKey(url)) {
- // Item is already downloaded/downloading, skip it.
- logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
- return;
- }
- itemsPending.put(url, saveAs);
- threadPool.addThread(new DownloadFileThread(url, saveAs, this));
- }
+ public abstract void addURLToDownload(URL url, File saveAs);
/**
* Queues file to be downloaded and saved. With options.
@@ -192,67 +176,30 @@ public abstract class AbstractRipper
* @param saveAs
* Where the downloaded file is stored.
*/
- public void downloadCompleted(URL url, File saveAs) {
- if (observer == null) {
- return;
- }
- try {
- String path = Utils.removeCWD(saveAs);
- RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path);
- itemsPending.remove(url);
- itemsCompleted.put(url, saveAs);
- observer.update(this, msg);
-
- checkIfComplete();
- } catch (Exception e) {
- logger.error("Exception while updating observer: ", e);
- }
- }
-
+ public abstract void downloadCompleted(URL url, File saveAs);
/**
* Notifies observers that a file could not be downloaded (includes a reason).
* @param url
* @param reason
*/
- public void downloadErrored(URL url, String reason) {
- if (observer == null) {
- return;
- }
- itemsPending.remove(url);
- itemsErrored.put(url, reason);
- observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason));
-
- checkIfComplete();
- }
-
+ public abstract void downloadErrored(URL url, String reason);
/**
* Notify observers that a download could not be completed,
* but was not technically an "error".
* @param url
* @param message
*/
- public void downloadProblem(URL url, String message) {
- if (observer == null) {
- return;
- }
-
- itemsPending.remove(url);
- itemsErrored.put(url, message);
- observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message));
-
-
- checkIfComplete();
- }
+ public abstract void downloadProblem(URL url, String message);
/**
* Notifies observers and updates state if all files have been ripped.
*/
- private void checkIfComplete() {
+ protected void checkIfComplete() {
if (observer == null) {
return;
}
- if (!completed && itemsPending.isEmpty()) {
+ if (!completed) {
completed = true;
logger.info(" Rip completed!");
@@ -274,26 +221,7 @@ public abstract class AbstractRipper
return workingDir;
}
- /**
- * Sets directory to save all ripped files to.
- * @param url
- * URL to define how the workin directory should be saved.
- */
- public void setWorkingDir(URL url) throws IOException {
- String path = Utils.getWorkingDirectory().getCanonicalPath();
- if (!path.endsWith(File.separator)) {
- path += File.separator;
- }
- String title = getAlbumTitle(this.url);
- title = Utils.filesystemSafe(title);
- path += title + File.separator;
- this.workingDir = new File(path);
- if (!this.workingDir.exists()) {
- logger.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir));
- this.workingDir.mkdirs();
- }
- logger.debug("Set working directory to: " + this.workingDir);
- }
+ public abstract void setWorkingDir(URL url) throws IOException;
public String getAlbumTitle(URL url) throws MalformedURLException {
return getHost() + "_" + getGID(url);
@@ -309,9 +237,17 @@ public abstract class AbstractRipper
* If no compatible rippers can be found.
*/
public static AbstractRipper getRipper(URL url) throws Exception {
- for (Constructor constructor : getRipperConstructors()) {
+ for (Constructor constructor : getRipperConstructors("com.rarchives.ripme.ripper.rippers")) {
try {
- AbstractRipper ripper = (AbstractRipper) constructor.newInstance(url);
+ AlbumRipper ripper = (AlbumRipper) constructor.newInstance(url);
+ return ripper;
+ } catch (Exception e) {
+ // Incompatible rippers *will* throw exceptions during instantiation.
+ }
+ }
+ for (Constructor constructor : getRipperConstructors("com.rarchives.ripme.ripper.rippers.video")) {
+ try {
+ VideoRipper ripper = (VideoRipper) constructor.newInstance(url);
return ripper;
} catch (Exception e) {
// Incompatible rippers *will* throw exceptions during instantiation.
@@ -325,9 +261,9 @@ public abstract class AbstractRipper
* List of constructors for all eligible Rippers.
* @throws Exception
*/
- private static List> getRipperConstructors() throws Exception {
+ private static List> getRipperConstructors(String pkg) throws Exception {
List> constructors = new ArrayList>();
- for (Class clazz : Utils.getClassesForPackage("com.rarchives.ripme.ripper.rippers")) {
+ for (Class clazz : Utils.getClassesForPackage(pkg)) {
if (AbstractRipper.class.isAssignableFrom(clazz)) {
constructors.add( (Constructor) clazz.getConstructor(URL.class) );
}
@@ -347,28 +283,9 @@ public abstract class AbstractRipper
observer.update(this, new RipStatusMessage(status, message));
}
- /**
- * @return
- * Integer between 0 and 100 defining the progress of the album rip.
- */
- public int getCompletionPercentage() {
- double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size();
- return (int) (100 * ( (total - itemsPending.size()) / total));
- }
+ public abstract int getCompletionPercentage();
- /**
- * @return
- * Human-readable information on the status of the current rip.
- */
- public String getStatusText() {
- StringBuilder sb = new StringBuilder();
- sb.append(getCompletionPercentage())
- .append("% ")
- .append("- Pending: " ).append(itemsPending.size())
- .append(", Completed: ").append(itemsCompleted.size())
- .append(", Errored: " ).append(itemsErrored.size());
- return sb.toString();
- }
+ public abstract String getStatusText();
/**
* Rips the album when the thread is invoked.
@@ -382,4 +299,10 @@ public abstract class AbstractRipper
}
}
+ public void setBytesTotal(int bytes) {
+ // Do nothing
+ }
+ public void setBytesCompleted(int bytes) {
+ // Do nothing
+ }
}
\ No newline at end of file
diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java
new file mode 100644
index 00000000..ca456737
--- /dev/null
+++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java
@@ -0,0 +1,146 @@
+package com.rarchives.ripme.ripper;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.rarchives.ripme.ui.RipStatusMessage;
+import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
+import com.rarchives.ripme.utils.Utils;
+
+public abstract class AlbumRipper extends AbstractRipper {
+
+ protected Map itemsPending = Collections.synchronizedMap(new HashMap());
+ protected Map itemsCompleted = Collections.synchronizedMap(new HashMap());
+ protected Map itemsErrored = Collections.synchronizedMap(new HashMap());
+
+ public AlbumRipper(URL url) throws IOException {
+ super(url);
+ }
+
+ public abstract boolean canRip(URL url);
+ public abstract URL sanitizeURL(URL url) throws MalformedURLException;
+ public abstract void rip() throws IOException;
+ public abstract String getHost();
+ public abstract String getGID(URL url) throws MalformedURLException;
+
+ @Override
+ public void addURLToDownload(URL url, File saveAs) {
+ if (itemsPending.containsKey(url)
+ || itemsCompleted.containsKey(url)
+ || itemsErrored.containsKey(url)) {
+ // Item is already downloaded/downloading, skip it.
+ logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
+ return;
+ }
+ itemsPending.put(url, saveAs);
+ threadPool.addThread(new DownloadFileThread(url, saveAs, this));
+ }
+
+ @Override
+ public void downloadCompleted(URL url, File saveAs) {
+ if (observer == null) {
+ return;
+ }
+ try {
+ String path = Utils.removeCWD(saveAs);
+ RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path);
+ itemsPending.remove(url);
+ itemsCompleted.put(url, saveAs);
+ observer.update(this, msg);
+
+ checkIfComplete();
+ } catch (Exception e) {
+ logger.error("Exception while updating observer: ", e);
+ }
+ }
+
+ @Override
+ public void downloadErrored(URL url, String reason) {
+ if (observer == null) {
+ return;
+ }
+ itemsPending.remove(url);
+ itemsErrored.put(url, reason);
+ observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason));
+
+ checkIfComplete();
+ }
+
+ @Override
+ public void downloadProblem(URL url, String message) {
+ if (observer == null) {
+ return;
+ }
+
+ itemsPending.remove(url);
+ itemsErrored.put(url, message);
+ observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message));
+
+ checkIfComplete();
+ }
+
+ /**
+ * Notifies observers and updates state if all files have been ripped.
+ */
+ @Override
+ protected void checkIfComplete() {
+ if (observer == null) {
+ return;
+ }
+ if (itemsPending.isEmpty()) {
+ super.checkIfComplete();
+ }
+ }
+
+ /**
+ * Sets directory to save all ripped files to.
+ * @param url
+ * URL to define how the workin directory should be saved.
+ */
+ @Override
+ public void setWorkingDir(URL url) throws IOException {
+ String path = Utils.getWorkingDirectory().getCanonicalPath();
+ if (!path.endsWith(File.separator)) {
+ path += File.separator;
+ }
+ String title = getAlbumTitle(this.url);
+ title = Utils.filesystemSafe(title);
+ path += title + File.separator;
+ this.workingDir = new File(path);
+ if (!this.workingDir.exists()) {
+ logger.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir));
+ this.workingDir.mkdirs();
+ }
+ logger.debug("Set working directory to: " + this.workingDir);
+ }
+
+ /**
+ * @return
+ * Integer between 0 and 100 defining the progress of the album rip.
+ */
+ @Override
+ public int getCompletionPercentage() {
+ double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size();
+ return (int) (100 * ( (total - itemsPending.size()) / total));
+ }
+
+ /**
+ * @return
+ * Human-readable information on the status of the current rip.
+ */
+ @Override
+ public String getStatusText() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getCompletionPercentage())
+ .append("% ")
+ .append("- Pending: " ).append(itemsPending.size())
+ .append(", Completed: ").append(itemsCompleted.size())
+ .append(", Errored: " ).append(itemsErrored.size());
+ return sb.toString();
+ }
+}
diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java
new file mode 100644
index 00000000..d50cf31b
--- /dev/null
+++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java
@@ -0,0 +1,126 @@
+package com.rarchives.ripme.ripper;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+import org.apache.log4j.Logger;
+
+import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
+import com.rarchives.ripme.utils.Utils;
+
+/**
+ * Thread for downloading files.
+ * Includes retry logic, observer notifications, and other goodies.
+ */
+public class DownloadVideoThread extends Thread {
+
+ private static final Logger logger = Logger.getLogger(DownloadVideoThread.class);
+
+ private URL url;
+ private File saveAs;
+ private String prettySaveAs;
+ private AbstractRipper observer;
+ private int retries;
+
+ public DownloadVideoThread(URL url, File saveAs, AbstractRipper observer) {
+ super();
+ this.url = url;
+ this.saveAs = saveAs;
+ this.prettySaveAs = Utils.removeCWD(saveAs);
+ this.observer = observer;
+ this.retries = Utils.getConfigInteger("download.retries", 1);
+ }
+
+ /**
+ * Attempts to download the file. Retries as needed.
+ * Notifies observers upon completion/error/warn.
+ */
+ public void run() {
+ try {
+ observer.stopCheck();
+ } catch (IOException e) {
+ observer.downloadErrored(url, "Download interrupted");
+ return;
+ }
+ if (saveAs.exists()) {
+ if (Utils.getConfigBoolean("file.overwrite", false)) {
+ logger.info("[!] Deleting existing file" + prettySaveAs);
+ saveAs.delete();
+ } else {
+ logger.info("[!] Skipping " + url + " -- file already exists: " + prettySaveAs);
+ observer.downloadProblem(url, "File already exists: " + prettySaveAs);
+ return;
+ }
+ }
+
+ int bytesTotal, bytesDownloaded = 0;
+ try {
+ bytesTotal = getTotalBytes(this.url);
+ } catch (IOException e) {
+ logger.error("Failed to get file size at " + this.url, e);
+ observer.downloadErrored(this.url, "Failed to get file size of " + this.url);
+ return;
+ }
+ observer.setBytesTotal(bytesTotal);
+ observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal);
+ logger.info("Size of file at " + this.url + " = " + bytesTotal + "b");
+
+ int tries = 0; // Number of attempts to download
+ do {
+ InputStream bis = null; OutputStream fos = null;
+ byte[] data = new byte[1024]; int bytesRead;
+ try {
+ logger.info(" Downloading file: " + url + (tries > 0 ? " Retry #" + tries : ""));
+ observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm());
+ tries += 1;
+ bis = new BufferedInputStream(this.url.openStream());
+ fos = new FileOutputStream(saveAs);
+ while ( (bytesRead = bis.read(data)) != -1) {
+ try {
+ observer.stopCheck();
+ } catch (IOException e) {
+ observer.downloadErrored(url, "Download interrupted");
+ return;
+ }
+ fos.write(data, 0, bytesRead);
+ bytesDownloaded += bytesRead;
+ observer.setBytesCompleted(bytesDownloaded);
+ observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded);
+ }
+ bis.close();
+ fos.close();
+ break; // Download successful: break out of infinite loop
+ } catch (IOException e) {
+ logger.error("[!] Exception while downloading file: " + url + " - " + e.getMessage(), e);
+ } finally {
+ // Close any open streams
+ try {
+ if (bis != null) { bis.close(); }
+ } catch (IOException e) { }
+ try {
+ if (fos != null) { fos.close(); }
+ } catch (IOException e) { }
+ }
+ if (tries > this.retries) {
+ logger.error("[!] Exceeded maximum retries (" + this.retries + ") for URL " + url);
+ observer.downloadErrored(url, "Failed to download " + url.toExternalForm());
+ return;
+ }
+ } while (true);
+ observer.downloadCompleted(url, saveAs);
+ logger.info("[+] Saved " + url + " as " + this.prettySaveAs);
+ }
+
+ private int getTotalBytes(URL url) throws IOException {
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.setRequestMethod("HEAD");
+ return conn.getContentLength();
+ }
+
+}
\ No newline at end of file
diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java
new file mode 100644
index 00000000..ae52e0c3
--- /dev/null
+++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java
@@ -0,0 +1,121 @@
+package com.rarchives.ripme.ripper;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+import com.rarchives.ripme.ui.RipStatusMessage;
+import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
+import com.rarchives.ripme.utils.Utils;
+
+public abstract class VideoRipper extends AbstractRipper {
+
+ private int bytesTotal = 1,
+ bytesCompleted = 1;
+
+ public VideoRipper(URL url) throws IOException {
+ super(url);
+ }
+
+ public abstract boolean canRip(URL url);
+ public abstract void rip() throws IOException;
+ public abstract String getHost();
+ public abstract String getGID(URL url) throws MalformedURLException;
+
+ @Override
+ public void setBytesTotal(int bytes) {
+ this.bytesTotal = bytes;
+ }
+ @Override
+ public void setBytesCompleted(int bytes) {
+ this.bytesCompleted = bytes;
+ }
+
+ @Override
+ public void addURLToDownload(URL url, File saveAs) {
+ threadPool.addThread(new DownloadVideoThread(url, saveAs, this));
+ }
+
+ @Override
+ public void setWorkingDir(URL url) throws IOException {
+ String path = Utils.getWorkingDirectory().getCanonicalPath();
+ if (!path.endsWith(File.separator)) {
+ path += File.separator;
+ }
+ path += "videos" + File.separator;
+ this.workingDir = new File(path);
+ if (!this.workingDir.exists()) {
+ logger.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir));
+ this.workingDir.mkdirs();
+ }
+ logger.debug("Set working directory to: " + this.workingDir);
+ }
+
+ @Override
+ public int getCompletionPercentage() {
+ return (int) (100 * (bytesCompleted / (float) bytesTotal));
+ }
+
+ @Override
+ public void downloadCompleted(URL url, File saveAs) {
+ if (observer == null) {
+ return;
+ }
+ try {
+ String path = Utils.removeCWD(saveAs);
+ RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path);
+ observer.update(this, msg);
+
+ checkIfComplete();
+ } catch (Exception e) {
+ logger.error("Exception while updating observer: ", e);
+ }
+ }
+ @Override
+ public void downloadErrored(URL url, String reason) {
+ if (observer == null) {
+ return;
+ }
+ observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason));
+ checkIfComplete();
+ }
+ @Override
+ public void downloadProblem(URL url, String message) {
+ if (observer == null) {
+ return;
+ }
+ observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " : " + message));
+ checkIfComplete();
+ }
+
+ @Override
+ public String getStatusText() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getCompletionPercentage())
+ .append("% ")
+ .append(" - ")
+ .append(Utils.bytesToHumanReadable(bytesCompleted))
+ .append(" / ")
+ .append(Utils.bytesToHumanReadable(bytesTotal));
+ return sb.toString();
+ }
+
+ @Override
+ public URL sanitizeURL(URL url) throws MalformedURLException {
+ return url;
+ }
+
+ /**
+ * Notifies observers and updates state if all files have been ripped.
+ */
+ @Override
+ protected void checkIfComplete() {
+ if (observer == null) {
+ return;
+ }
+ if (bytesCompleted >= bytesTotal) {
+ super.checkIfComplete();
+ }
+ }
+}
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
index 71bf73fe..8f10a3ff 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
@@ -13,9 +13,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
-public class ChanRipper extends AbstractRipper {
+public class ChanRipper extends AlbumRipper {
private static final Logger logger = Logger.getLogger(ChanRipper.class);
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
index f37b347e..d0fc3f98 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
@@ -18,10 +18,10 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.Utils;
-public class DeviantartRipper extends AbstractRipper {
+public class DeviantartRipper extends AlbumRipper {
private static final String DOMAIN = "deviantart.com",
HOST = "deviantart";
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java
index 9d0559bb..b0ed61b2 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java
@@ -12,9 +12,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
-public class EightmusesRipper extends AbstractRipper {
+public class EightmusesRipper extends AlbumRipper {
private static final String DOMAIN = "8muses.com",
HOST = "8muses";
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GonewildRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GonewildRipper.java
index aac913d2..40a376ca 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/GonewildRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GonewildRipper.java
@@ -11,10 +11,10 @@ import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.Jsoup;
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.Utils;
-public class GonewildRipper extends AbstractRipper {
+public class GonewildRipper extends AlbumRipper {
private static final String HOST = "gonewild";
private static final Logger logger = Logger.getLogger(GonewildRipper.class);
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java
index 515e9443..c08b7167 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java
@@ -11,9 +11,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
-public class ImagearnRipper extends AbstractRipper {
+public class ImagearnRipper extends AlbumRipper {
private static final String DOMAIN = "imagearn.com",
HOST = "imagearn";
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java
index a69f8748..a37a0017 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java
@@ -11,9 +11,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
-public class ImagefapRipper extends AbstractRipper {
+public class ImagefapRipper extends AlbumRipper {
private static final String DOMAIN = "imagefap.com",
HOST = "imagefap";
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java
index b8ab02a9..2d9470c2 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java
@@ -18,11 +18,11 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Utils;
-public class ImgurRipper extends AbstractRipper {
+public class ImgurRipper extends AlbumRipper {
private static final String DOMAIN = "imgur.com",
HOST = "imgur";
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
index 718c57d7..3f39fd4f 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
@@ -13,9 +13,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
-public class InstagramRipper extends AbstractRipper {
+public class InstagramRipper extends AlbumRipper {
private static final String DOMAIN = "instagram.com",
HOST = "instagram";
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/KinkyshareRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/KinkyshareRipper.java
index 702fc537..dade912d 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/KinkyshareRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/KinkyshareRipper.java
@@ -11,9 +11,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
-public class KinkyshareRipper extends AbstractRipper {
+public class KinkyshareRipper extends AlbumRipper {
private static final String HOST = "kinkyshare";
private static final Logger logger = Logger.getLogger(KinkyshareRipper.class);
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java
index daf33c39..52bfb33f 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java
@@ -11,10 +11,10 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ripper.DownloadThreadPool;
-public class MotherlessRipper extends AbstractRipper {
+public class MotherlessRipper extends AlbumRipper {
private static final String DOMAIN = "motherless.com",
HOST = "motherless";
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java
index a6c03458..e12d894d 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java
@@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
+import java.net.SocketTimeoutException;
import java.net.URL;
import java.util.List;
import java.util.regex.Matcher;
@@ -14,12 +15,11 @@ import org.json.JSONTokener;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.RipUtils;
import com.rarchives.ripme.utils.Utils;
-import java.net.SocketTimeoutException;
-public class RedditRipper extends AbstractRipper {
+public class RedditRipper extends AlbumRipper {
public RedditRipper(URL url) throws IOException {
super(url);
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java
index 4ec4b649..86c815f1 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SeeniveRipper.java
@@ -12,10 +12,10 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.ripper.DownloadThreadPool;
-public class SeeniveRipper extends AbstractRipper {
+public class SeeniveRipper extends AlbumRipper {
private static final String DOMAIN = "seenive.com",
HOST = "seenive";
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java
index ee924f44..21f23bdf 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java
@@ -12,10 +12,10 @@ import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.Utils;
-public class TumblrRipper extends AbstractRipper {
+public class TumblrRipper extends AlbumRipper {
private static final String DOMAIN = "tumblr.com",
HOST = "tumblr";
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
index 810734cb..e5cae727 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java
@@ -16,10 +16,10 @@ import org.json.JSONTokener;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
import com.rarchives.ripme.utils.Utils;
-public class TwitterRipper extends AbstractRipper {
+public class TwitterRipper extends AlbumRipper {
private static final String DOMAIN = "twitter.com",
HOST = "twitter";
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java
index 96fadcf7..8ddb2e3c 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VineboxRipper.java
@@ -12,9 +12,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
-public class VineboxRipper extends AbstractRipper {
+public class VineboxRipper extends AlbumRipper {
private static final String DOMAIN = "vinebox.co",
HOST = "vinebox";
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java
index efcadaf7..a3654e9f 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java
@@ -18,9 +18,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
-public class VkRipper extends AbstractRipper {
+public class VkRipper extends AlbumRipper {
private static final String DOMAIN = "vk.com",
HOST = "vk";
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
index e692e5a2..5f05ee64 100644
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
@@ -11,9 +11,9 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
-import com.rarchives.ripme.ripper.AbstractRipper;
+import com.rarchives.ripme.ripper.AlbumRipper;
-public class XhamsterRipper extends AbstractRipper {
+public class XhamsterRipper extends AlbumRipper {
private static final String DOMAIN = "xhamster.com",
HOST = "xhamster";
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java
new file mode 100644
index 00000000..468c6691
--- /dev/null
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XvideosRipper.java
@@ -0,0 +1,84 @@
+package com.rarchives.ripme.ripper.rippers.video;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.log4j.Logger;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+import com.rarchives.ripme.ripper.VideoRipper;
+
+public class XvideosRipper extends VideoRipper {
+
+ private static final String HOST = "xvideos";
+ private static final Logger logger = Logger.getLogger(XvideosRipper.class);
+
+ public XvideosRipper(URL url) throws IOException {
+ super(url);
+ }
+
+ @Override
+ public String getHost() {
+ return HOST;
+ }
+
+ @Override
+ public boolean canRip(URL url) {
+ Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/video[0-9]+.*$");
+ Matcher m = p.matcher(url.toExternalForm());
+ return m.matches();
+ }
+
+ @Override
+ public String getAlbumTitle(URL url) {
+ return "videos";
+ }
+
+ @Override
+ public URL sanitizeURL(URL url) throws MalformedURLException {
+ return url;
+ }
+
+ @Override
+ public String getGID(URL url) throws MalformedURLException {
+ Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/video([0-9]+).*$");
+ Matcher m = p.matcher(url.toExternalForm());
+ if (m.matches()) {
+ return m.group(1);
+ }
+
+ throw new MalformedURLException(
+ "Expected xvideo format:"
+ + "xvideos.com/video####"
+ + " Got: " + url);
+ }
+
+ @Override
+ public void rip() throws IOException {
+ logger.info(" Retrieving " + this.url.toExternalForm());
+ Document doc = Jsoup.connect(this.url.toExternalForm())
+ .userAgent(USER_AGENT)
+ .get();
+ Elements embeds = doc.select("embed");
+ if (embeds.size() == 0) {
+ throw new IOException("Could not find Embed code at " + url);
+ }
+ Element embed = embeds.get(0);
+ String vars = embed.attr("flashvars");
+ for (String var : vars.split("&")) {
+ if (var.startsWith("flv_url=")) {
+ String vidUrl = var.substring("flv_url=".length());
+ vidUrl = URLDecoder.decode(vidUrl, "UTF-8");
+ addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url));
+ }
+ }
+ waitForThreads();
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java
index d83f07ad..ad2f0550 100644
--- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java
+++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java
@@ -721,6 +721,13 @@ public class MainWindow implements Runnable, RipStatusHandler {
}
});
mainFrame.pack();
+ break;
+ case COMPLETED_BYTES:
+ // Update completed bytes
+ break;
+ case TOTAL_BYTES:
+ // Update total bytes
+ break;
}
}
diff --git a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java
index 45c01edc..fee3fc59 100644
--- a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java
+++ b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java
@@ -11,7 +11,9 @@ public class RipStatusMessage {
DOWNLOAD_COMPLETE("Download Complete"),
DOWNLOAD_ERRORED("Download Errored"),
RIP_COMPLETE("Rip Complete"),
- DOWNLOAD_WARN("Download problem");
+ DOWNLOAD_WARN("Download problem"),
+ TOTAL_BYTES("Total bytes"),
+ COMPLETED_BYTES("Completed bytes");
String value;
STATUS(String value) {
diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
index 3bdb62eb..f1d24ad8 100644
--- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
+++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
@@ -19,7 +19,7 @@ import org.jsoup.nodes.Document;
public class UpdateUtils {
private static final Logger logger = Logger.getLogger(UpdateUtils.class);
- private static final String DEFAULT_VERSION = "1.0.19";
+ private static final String DEFAULT_VERSION = "1.0.20";
private static final String updateJsonURL = "http://rarchives.com/ripme.json";
private static final String updateJarURL = "http://rarchives.com/ripme.jar";
private static final String mainFileName = "ripme.jar";
diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java
index 8d20837d..2500984a 100644
--- a/src/main/java/com/rarchives/ripme/utils/Utils.java
+++ b/src/main/java/com/rarchives/ripme/utils/Utils.java
@@ -247,4 +247,15 @@ public class Utils {
.replaceAll("__", "_")
.replaceAll("_+$", "");
}
+
+ public static String bytesToHumanReadable(int bytes) {
+ float fbytes = (float) bytes;
+ String[] mags = new String[] {"", "k", "m", "g", "t"};
+ int magIndex = 0;
+ while (fbytes >= 1024) {
+ fbytes /= 1024;
+ magIndex++;
+ }
+ return String.format("%.2f%sb", fbytes, mags[magIndex]);
+ }
}
\ No newline at end of file
diff --git a/src/test/java/com/rarchives/ripme/tst/AppTest.java b/src/test/java/com/rarchives/ripme/tst/AppTest.java
index 630e46d9..77368a89 100644
--- a/src/test/java/com/rarchives/ripme/tst/AppTest.java
+++ b/src/test/java/com/rarchives/ripme/tst/AppTest.java
@@ -4,6 +4,8 @@ import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
+import com.rarchives.ripme.utils.Utils;
+
public class AppTest extends TestCase {
/**
* Create the test case
@@ -25,6 +27,7 @@ public class AppTest extends TestCase {
* Rigourous Test :-)
*/
public void testApp() {
+ System.err.println(Utils.bytesToHumanReadable(1023 * 5000));
assertTrue( true );
}
}
diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java
new file mode 100644
index 00000000..847700a9
--- /dev/null
+++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java
@@ -0,0 +1,32 @@
+package com.rarchives.ripme.tst.ripper.rippers;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.rarchives.ripme.ripper.rippers.video.XvideosRipper;
+
+public class VideoRippersTest extends RippersTest {
+
+ public void testXvideosRipper() throws IOException {
+ if (false && !DOWNLOAD_CONTENT) {
+ return;
+ }
+ List contentURLs = new ArrayList();
+ contentURLs.add(new URL("http://www.xvideos.com/video1428195/stephanie_first_time_anal"));
+ contentURLs.add(new URL("http://www.xvideos.com/video7136868/vid-20140205-wa0011"));
+ for (URL url : contentURLs) {
+ try {
+ XvideosRipper ripper = new XvideosRipper(url);
+ ripper.rip();
+ assert(ripper.getWorkingDir().listFiles().length > 1);
+ deleteDir(ripper.getWorkingDir());
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail("Error while ripping URL " + url + ": " + e.getMessage());
+ }
+ }
+ }
+
+}