Merge pull request #1190 from buzzlightmonth/codeClean

Removal of references to AlbumRipper

src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java

@@ -3,19 +3,21 @@ package com.rarchives.ripme.ripper.rippers;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 
-import com.rarchives.ripme.ripper.AlbumRipper;
+import com.rarchives.ripme.ripper.AbstractHTMLRipper;
 import com.rarchives.ripme.ripper.DownloadThreadPool;
 import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
 import com.rarchives.ripme.utils.Http;
 import com.rarchives.ripme.utils.Utils;
 
-public class MotherlessRipper extends AlbumRipper {
+public class MotherlessRipper extends AbstractHTMLRipper {
 
     private static final String DOMAIN = "motherless.com",
                                 HOST   = "motherless";
@@ -37,6 +39,52 @@ public class MotherlessRipper extends AlbumRipper {
         return url.getHost().endsWith(DOMAIN);
     }
 
+    @Override
+    protected String getDomain() {
+        return DOMAIN;
+    }
+
+    @Override
+    protected Document getFirstPage() throws IOException {
+        return Http.url(url).referrer("http://motherless.com").get();
+    }
+
+    @Override
+    protected List<String> getURLsFromPage(Document page) {
+        List<String> pageURLs = new ArrayList<>();
+
+        for (Element thumb : page.select("div.thumb a.img-container")) {
+            if (isStopped()) {
+                break;
+            }
+            String thumbURL = thumb.attr("href");
+            if (thumbURL.contains("pornmd.com")) {
+                continue;
+            }
+
+            String url;
+            if (!thumbURL.startsWith("http")) {
+                url = "http://" + DOMAIN + thumbURL;
+            } else {
+                url = thumbURL;
+            }
+            pageURLs.add(url);
+
+            if (isThisATest()) {
+                break;
+            }
+        }
+
+        return pageURLs;
+    }
+
+    @Override
+    protected void downloadURL(URL url, int index) {
+        // Create thread for finding image at "url" page
+        MotherlessImageThread mit = new MotherlessImageThread(url, index);
+        motherlessThreadPool.addThread(mit);
+    }
+
     @Override
     public String getHost() {
         return HOST;
@@ -77,34 +125,14 @@ public class MotherlessRipper extends AlbumRipper {
             }
             LOGGER.info("Retrieving " + nextURL);
             sendUpdate(STATUS.LOADING_RESOURCE, nextURL);
-            Document doc = Http.url(nextURL)
-                               .referrer("http://motherless.com")
-                               .get();
-            for (Element thumb : doc.select("div.thumb a.img-container")) {
-                if (isStopped()) {
-                    break;
-                }
-                String thumbURL = thumb.attr("href");
-                if (thumbURL.contains("pornmd.com")) {
-                    continue;
-                }
-                URL url;
-                if (!thumbURL.startsWith("http")) {
-                    url = new URL("http://" + DOMAIN + thumbURL);
-                }
-                else {
-                    url = new URL(thumbURL);
-                }
-                index += 1;
+            Document doc = getFirstPage();
+            List<String> URLs = getURLsFromPage(doc);
 
-                // Create thread for finding image at "url" page
-                MotherlessImageThread mit = new MotherlessImageThread(url, index);
-                motherlessThreadPool.addThread(mit);
-
-                if (isThisATest()) {
-                    break;
-                }
+            for (String url: URLs) {
+                downloadURL(new URL(url), index);
+                index ++;
             }
+
             if (isThisATest()) {
                 break;
             }

src/main/java/com/rarchives/ripme/ripper/rippers/NewsfilterRipper.java

@@ -10,14 +10,10 @@ import java.util.regex.Pattern;
 
 import com.rarchives.ripme.ripper.AbstractHTMLRipper;
 import com.rarchives.ripme.utils.Http;
-import org.jsoup.Connection;
-import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
 
-import com.rarchives.ripme.ripper.AlbumRipper;
-
 public class NewsfilterRipper extends AbstractHTMLRipper {
 
     private static final String HOST = "newsfilter";

src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java

@@ -13,11 +13,8 @@ import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
 
-import com.rarchives.ripme.ripper.AlbumRipper;
 import com.rarchives.ripme.ripper.DownloadThreadPool;
-import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
 import com.rarchives.ripme.utils.Http;
-import com.rarchives.ripme.utils.Utils;
 
 public class NfsfwRipper extends AbstractHTMLRipper {
 

src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java

@@ -14,9 +14,7 @@ import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
 
-import com.rarchives.ripme.ripper.AlbumRipper;
 import com.rarchives.ripme.ripper.DownloadThreadPool;
-import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
 import com.rarchives.ripme.utils.Http;
 import com.rarchives.ripme.utils.Utils;
 

src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java

@@ -3,33 +3,171 @@ package com.rarchives.ripme.ripper.rippers;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.rarchives.ripme.ripper.AbstractJSONRipper;
 import org.json.JSONArray;
 import org.json.JSONObject;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 
-import com.rarchives.ripme.ripper.AlbumRipper;
 import com.rarchives.ripme.utils.Http;
 import com.rarchives.ripme.utils.Utils;
 
-public class VkRipper extends AlbumRipper {
+public class VkRipper extends AbstractJSONRipper {
 
     private static final String DOMAIN = "vk.com",
                                 HOST   = "vk";
 
+    enum RipType { VIDEO, IMAGE }
+
+    private RipType RIP_TYPE;
+    private String oid;
+
     public VkRipper(URL url) throws IOException {
         super(url);
     }
 
+    @Override
+    public String getHost() {
+        return HOST;
+    }
+
+    @Override
+    protected String getDomain() {
+        return DOMAIN;
+    }
+
+    @Override
+    protected JSONObject getFirstPage() throws IOException {
+        if (RIP_TYPE == RipType.VIDEO) {
+            oid = getGID(this.url).replace("videos", "");
+            String u = "http://vk.com/al_video.php";
+            Map<String, String> postData = new HashMap<>();
+            postData.put("al", "1");
+            postData.put("act", "load_videos_silent");
+            postData.put("offset", "0");
+            postData.put("oid", oid);
+            Document doc = Http.url(u)
+                    .referrer(this.url)
+                    .ignoreContentType()
+                    .data(postData)
+                    .post();
+            String[] jsonStrings = doc.toString().split("<!>");
+            return new JSONObject(jsonStrings[jsonStrings.length - 1]);
+        } else {
+            Map<String,String> photoIDsToURLs = new HashMap<>();
+            int offset = 0;
+            while (true) {
+                LOGGER.info("    Retrieving " + this.url);
+                Map<String,String> postData = new HashMap<>();
+                postData.put("al", "1");
+                postData.put("offset", Integer.toString(offset));
+                postData.put("part", "1");
+                Document doc = Http.url(this.url)
+                        .referrer(this.url)
+                        .ignoreContentType()
+                        .data(postData)
+                        .post();
+
+                String body = doc.toString();
+                if (!body.contains("<div")) {
+                    break;
+                }
+                body = body.substring(body.indexOf("<div"));
+                doc = Jsoup.parseBodyFragment(body);
+                List<Element> elements = doc.select("a");
+                Set<String> photoIDsToGet = new HashSet<>();
+                for (Element a : elements) {
+                    if (!a.attr("onclick").contains("showPhoto('")) {
+                        LOGGER.error("a: " + a);
+                        continue;
+                    }
+                    String photoID = a.attr("onclick");
+                    photoID = photoID.substring(photoID.indexOf("showPhoto('") + "showPhoto('".length());
+                    photoID = photoID.substring(0, photoID.indexOf("'"));
+                    if (!photoIDsToGet.contains(photoID)) {
+                        photoIDsToGet.add(photoID);
+                    }
+                }
+                for (String photoID : photoIDsToGet) {
+                    if (!photoIDsToURLs.containsKey(photoID)) {
+                        try {
+                            photoIDsToURLs.putAll(getPhotoIDsToURLs(photoID));
+                        } catch (IOException e) {
+                            LOGGER.error("Exception while retrieving photo id " + photoID, e);
+                            continue;
+                        }
+                    }
+                    if (!photoIDsToURLs.containsKey(photoID)) {
+                        LOGGER.error("Could not find URL for photo ID: " + photoID);
+                        continue;
+                    }
+                    if (isStopped() || isThisATest()) {
+                        break;
+                    }
+                }
+
+                if (elements.size() < 40 || isStopped() || isThisATest()) {
+                    break;
+                }
+                offset += elements.size();
+            }
+            // Slight hack to make this into effectively a JSON ripper
+            return new JSONObject(photoIDsToURLs);
+        }
+    }
+
+    @Override
+    protected List<String> getURLsFromJSON(JSONObject page) {
+        List<String> pageURLs = new ArrayList<>();
+        if (RIP_TYPE == RipType.VIDEO) {
+            JSONArray videos = page.getJSONArray("all");
+            LOGGER.info("Found " + videos.length() + " videos");
+
+            for (int i = 0; i < videos.length(); i++) {
+                JSONArray jsonVideo = videos.getJSONArray(i);
+                int vidid = jsonVideo.getInt(1);
+                String videoURL;
+                try {
+                    videoURL = com.rarchives.ripme.ripper.rippers.video.VkRipper.getVideoURLAtPage(
+                            "http://vk.com/video" + oid + "_" + vidid);
+                } catch (IOException e) {
+                    LOGGER.error("Error while ripping video id: " + vidid);
+                    return pageURLs;
+                }
+                pageURLs.add(videoURL);
+            }
+        } else {
+            Iterator keys = page.keys();
+            while (keys.hasNext()) {
+                pageURLs.add(page.getString((String) keys.next()));
+            }
+        }
+        return pageURLs;
+    }
+
+    @Override
+    protected void downloadURL(URL url, int index) {
+        if (RIP_TYPE == RipType.VIDEO) {
+            String prefix = "";
+            if (Utils.getConfigBoolean("download.save_order", true)) {
+                prefix = String.format("%03d_", index + 1);
+            }
+            addURLToDownload(url, prefix);
+            try {
+                Thread.sleep(500);
+            } catch (InterruptedException e) {
+                LOGGER.error("Interrupted while waiting to fetch next video URL", e);
+            }
+        } else {
+            addURLToDownload(url);
+        }
+    }
+
     @Override
     public boolean canRip(URL url) {
         if (!url.getHost().endsWith(DOMAIN)) {
@@ -48,115 +186,19 @@ public class VkRipper extends AlbumRipper {
     @Override
     public void rip() throws IOException {
         if (this.url.toExternalForm().contains("/videos")) {
-            ripVideos();
+            RIP_TYPE = RipType.VIDEO;
+            JSONObject json = getFirstPage();
+            List<String> URLs = getURLsFromJSON(json);
+            for (int index = 0; index < URLs.size(); index ++) {
+                downloadURL(new URL(URLs.get(index)), index);
+            }
+            waitForThreads();
         }
         else {
-            ripImages();
+            RIP_TYPE = RipType.IMAGE;
         }
     }
 
-    private void ripVideos() throws IOException {
-        String oid = getGID(this.url).replace("videos", "");
-        String u = "http://vk.com/al_video.php";
-        Map<String,String> postData = new HashMap<>();
-        postData.put("al", "1");
-        postData.put("act", "load_videos_silent");
-        postData.put("offset", "0");
-        postData.put("oid", oid);
-        Document doc = Http.url(u)
-                           .referrer(this.url)
-                           .ignoreContentType()
-                           .data(postData)
-                           .post();
-        String[] jsonStrings = doc.toString().split("<!>");
-        JSONObject json = new JSONObject(jsonStrings[jsonStrings.length - 1]);
-        JSONArray videos = json.getJSONArray("all");
-        LOGGER.info("Found " + videos.length() + " videos");
-        for (int i = 0; i < videos.length(); i++) {
-            JSONArray jsonVideo = videos.getJSONArray(i);
-            int vidid = jsonVideo.getInt(1);
-            String videoURL = com.rarchives.ripme.ripper.rippers.video.VkRipper.getVideoURLAtPage(
-                    "http://vk.com/video" + oid + "_" + vidid);
-            String prefix = "";
-            if (Utils.getConfigBoolean("download.save_order", true)) {
-                prefix = String.format("%03d_", i + 1);
-            }
-            addURLToDownload(new URL(videoURL), prefix);
-            try {
-                Thread.sleep(500);
-            } catch (InterruptedException e) {
-                LOGGER.error("Interrupted while waiting to fetch next video URL", e);
-                break;
-            }
-        }
-        waitForThreads();
-    }
-
-    private void ripImages() throws IOException {
-        Map<String,String> photoIDsToURLs = new HashMap<>();
-        int offset = 0;
-        while (true) {
-            LOGGER.info("    Retrieving " + this.url);
-
-            // al=1&offset=80&part=1
-            Map<String,String> postData = new HashMap<>();
-            postData.put("al", "1");
-            postData.put("offset", Integer.toString(offset));
-            postData.put("part", "1");
-            Document doc = Http.url(this.url)
-                               .referrer(this.url)
-                               .ignoreContentType()
-                               .data(postData)
-                               .post();
-
-            String body = doc.toString();
-            if (!body.contains("<div")) {
-                break;
-            }
-            body = body.substring(body.indexOf("<div"));
-            doc = Jsoup.parseBodyFragment(body);
-            List<Element> elements = doc.select("a");
-            Set<String> photoIDsToGet = new HashSet<>();
-            for (Element a : elements) {
-                if (!a.attr("onclick").contains("showPhoto('")) {
-                    LOGGER.error("a: " + a);
-                    continue;
-                }
-                String photoID = a.attr("onclick");
-                photoID = photoID.substring(photoID.indexOf("showPhoto('") + "showPhoto('".length());
-                photoID = photoID.substring(0, photoID.indexOf("'"));
-                if (!photoIDsToGet.contains(photoID)) {
-                    photoIDsToGet.add(photoID);
-                }
-            }
-            for (String photoID : photoIDsToGet) {
-                if (!photoIDsToURLs.containsKey(photoID)) {
-                    try {
-                        photoIDsToURLs.putAll(getPhotoIDsToURLs(photoID));
-                    } catch (IOException e) {
-                        LOGGER.error("Exception while retrieving photo id " + photoID, e);
-                        continue;
-                    }
-                }
-                if (!photoIDsToURLs.containsKey(photoID)) {
-                    LOGGER.error("Could not find URL for photo ID: " + photoID);
-                    continue;
-                }
-                String url = photoIDsToURLs.get(photoID);
-                addURLToDownload(new URL(url));
-                if (isStopped() || isThisATest()) {
-                    break;
-                }
-            }
-
-            if (elements.size() < 40 || isStopped() || isThisATest()) {
-                break;
-            }
-            offset += elements.size();
-        }
-        waitForThreads();
-    }
-
     private Map<String,String> getPhotoIDsToURLs(String photoID) throws IOException {
         Map<String,String> photoIDsToURLs = new HashMap<>();
         Map<String,String> postData = new HashMap<>();
@@ -191,11 +233,6 @@ public class VkRipper extends AlbumRipper {
         return photoIDsToURLs;
     }
 
-    @Override
-    public String getHost() {
-        return HOST;
-    }
-
     @Override
     public String getGID(URL url) throws MalformedURLException {
         Pattern p = Pattern.compile("^https?://(www\\.)?vk\\.com/(photos|album|videos)-?([a-zA-Z0-9_]+).*$");