Merge pull request #1040 from PeterSzakacs/issue/423

Ported the Teebplanet, nfsfw, pornhub, photobucket rippers to AbstractHTMLRipper

src/main/java/com/rarchives/ripme/ripper/rippers/NewsfilterRipper.java

@@ -3,9 +3,13 @@ package com.rarchives.ripme.ripper.rippers;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.rarchives.ripme.ripper.AbstractHTMLRipper;
+import com.rarchives.ripme.utils.Http;
 import org.jsoup.Connection;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
@@ -14,21 +18,15 @@ import org.jsoup.select.Elements;
 
 import com.rarchives.ripme.ripper.AlbumRipper;
 
-public class NewsfilterRipper extends AlbumRipper {
+public class NewsfilterRipper extends AbstractHTMLRipper {
+
     private static final String HOST = "newsfilter";
+    private static final String DOMAIN = "newsfilter.org";
 
     public NewsfilterRipper(URL url) throws IOException {
         super(url);
     }
 
-    @Override
-    public boolean canRip(URL url) {
-        //http://newsfilter.org/gallery/he-doubted-she-would-fuck-on-cam-happy-to-be-proven-wrong-216799
-        Pattern p = Pattern.compile("^https?://([wm]+\\.)?newsfilter\\.org/gallery/.+$");
-        Matcher m = p.matcher(url.toExternalForm());
-        return m.matches();
-    }
-
     @Override
     public URL sanitizeURL(URL url) throws MalformedURLException {
         String u = url.toExternalForm();
@@ -39,35 +37,6 @@ public class NewsfilterRipper extends AlbumRipper {
         return new URL(u);
     }
 
-    @Override
-    public void rip() throws IOException {
-        String gid = getGID(this.url);
-        String theurl = "http://newsfilter.org/gallery/" + gid;
-        LOGGER.info("Loading " + theurl);
-
-        Connection.Response resp = Jsoup.connect(theurl)
-            .timeout(5000)
-            .referrer("")
-            .userAgent(USER_AGENT)
-            .method(Connection.Method.GET)
-            .execute();
-        Document doc = resp.parse();
-
-        Elements thumbnails = doc.select("#galleryImages .inner-block img");
-        for (Element thumb : thumbnails) {
-            String thumbUrl = thumb.attr("src");
-            String picUrl = thumbUrl.replace("thumbs/", "");
-            addURLToDownload(new URL(picUrl));
-        }
-
-        waitForThreads();
-    }
-
-    @Override
-    public String getHost() {
-        return HOST;
-    }
-
     @Override
     public String getGID(URL url) throws MalformedURLException {
         Pattern p = Pattern.compile("^https?://([wm]+\\.)?newsfilter\\.org/gallery/([^/]+)$");
@@ -79,4 +48,37 @@ public class NewsfilterRipper extends AlbumRipper {
                 "Expected newsfilter gallery format: http://newsfilter.org/gallery/galleryid" +
                         " Got: " + url);
     }
+
+    @Override
+    public String getHost() {
+        return HOST;
+    }
+
+    @Override
+    protected String getDomain() {
+        return DOMAIN;
+    }
+
+    @Override
+    protected Document getFirstPage() throws IOException {
+        return Http.url(url).get();
+    }
+
+    @Override
+    protected List<String> getURLsFromPage(Document page) {
+        List<String> imgURLs = new ArrayList<>();
+        Elements thumbnails = page.select("#galleryImages .inner-block img");
+        for (Element thumb : thumbnails) {
+            String thumbUrl = thumb.attr("src");
+            String picUrl = thumbUrl.replace("thumbs/", "");
+            // use HTTP instead of HTTPS (less headaches)
+            imgURLs.add(picUrl.replaceFirst("https://", "http://"));
+        }
+        return imgURLs;
+    }
+
+    @Override
+    protected void downloadURL(URL url, int index) {
+        addURLToDownload(url, getPrefix(index));
+    }
 }

src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java

@@ -8,6 +8,7 @@ import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.rarchives.ripme.ripper.AbstractHTMLRipper;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
@@ -18,13 +19,22 @@ import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
 import com.rarchives.ripme.utils.Http;
 import com.rarchives.ripme.utils.Utils;
 
-public class NfsfwRipper extends AlbumRipper {
+public class NfsfwRipper extends AbstractHTMLRipper {
 
     private static final String DOMAIN = "nfsfw.com",
                                 HOST   = "nfsfw";
 
-    private Document albumDoc = null;
 
+    private int index = 0;
+    private String currentDir = "";
+    private List<String> subalbumURLs = new ArrayList<>();
+    private Pattern subalbumURLPattern = Pattern.compile(
+            "https?://[wm.]*nfsfw.com/gallery/v/[^/]+/(.+)$"
+    );
+
+    // cached first page
+    private Document fstPage;
+    // threads pool for downloading images from image pages
     private DownloadThreadPool nfsfwThreadPool;
 
     public NfsfwRipper(URL url) throws IOException {
@@ -32,39 +42,104 @@ public class NfsfwRipper extends AlbumRipper {
         nfsfwThreadPool = new DownloadThreadPool("NFSFW");
     }
 
+    @Override
+    protected String getDomain() {
+        return DOMAIN;
+    }
+
     @Override
     public String getHost() {
         return HOST;
     }
 
     @Override
-    public URL sanitizeURL(URL url) throws MalformedURLException {
+    protected Document getFirstPage() throws IOException {
-        return url;
+        // cache the first page
+        this.fstPage = Http.url(url).get();
+        return fstPage;
     }
 
     @Override
-    public String getAlbumTitle(URL url) throws MalformedURLException {
+    public Document getNextPage(Document page) throws IOException {
+        String nextURL = null;
+        Elements a = page.select("a.next");
+        if (!a.isEmpty()){
+            // Get next page of current album
+            nextURL = "http://nfsfw.com" + a.first().attr("href");
+        } else if (!subalbumURLs.isEmpty()){
+            // Get next sub-album
+            nextURL = subalbumURLs.remove(0);
+            LOGGER.info("Detected subalbum URL at:" + nextURL);
+            Matcher m = subalbumURLPattern.matcher(nextURL);
+            if (m.matches()) {
+                // Set the new save directory and save images with a new index
+                this.currentDir = m.group(1);
+                this.index = 0;
+            } else {
+                LOGGER.error("Invalid sub-album URL: " + nextURL);
+                nextURL = null;
+            }
+        }
+        // Wait
         try {
-            // Attempt to use album title as GID
+            Thread.sleep(2000);
-            if (albumDoc == null) {
+        } catch (InterruptedException e) {
-                albumDoc = Http.url(url).get();
+            LOGGER.error("Interrupted while waiting to load next page", e);
         }
-            String title = albumDoc.select("h2").first().text().trim();
+        if (nextURL == null){
-            return "nfsfw_" + Utils.filesystemSafe(title);
+            throw new IOException("No more pages");
-        } catch (Exception e) {
+        } else {
-            // Fall back to default album naming convention
+            return Http.url(nextURL).get();
+        }
+    }
+
+    @Override
+    protected List<String> getURLsFromPage(Document page) {
+        List<String> imagePageURLs = getImagePageURLs(page);
+
+        // Check if any sub-albums are present on this page
+        List<String> subalbumURLs = getSubalbumURLs(page);
+        this.subalbumURLs.addAll(subalbumURLs);
+
+        return imagePageURLs;
+    }
+
+    @Override
+    protected void downloadURL(URL url, int index) {
+        // if we are now downloading a sub-album, all images in it
+        // should be indexed starting from 0
+        if (!this.currentDir.equals("")){
+            index = ++this.index;
+        }
+        NfsfwImageThread t = new NfsfwImageThread(url, currentDir, index);
+        nfsfwThreadPool.addThread(t);
+    }
+
+    @Override
+    public URL sanitizeURL(URL url) throws MalformedURLException {
+        // always start on the first page of an album
+        // (strip the options after the '?')
+        String u = url.toExternalForm();
+        if (u.contains("?")) {
+            u = u.substring(0, u.indexOf("?"));
+            return new URL(u);
+        } else {
+            return url;
         }
-        return super.getAlbumTitle(url);
     }
 
     @Override
     public String getGID(URL url) throws MalformedURLException {
         Pattern p; Matcher m;
 
-        p = Pattern.compile("https?://[wm.]*nfsfw.com/gallery/v/([a-zA-Z0-9\\-_]+).*");
+        p = Pattern.compile("https?://[wm.]*nfsfw.com/gallery/v/(.*)$");
         m = p.matcher(url.toExternalForm());
         if (m.matches()) {
-            return m.group(1);
+            String group = m.group(1);
+            if (group.endsWith("/")) {
+                group = group.substring(0, group.length() - 1);
+            }
+            return group.replaceAll("/", "__");
         }
 
         throw new MalformedURLException(
@@ -74,75 +149,51 @@ public class NfsfwRipper extends AlbumRipper {
     }
 
     @Override
-    public void rip() throws IOException {
+    public DownloadThreadPool getThreadPool() {
-        List<Pair> subAlbums = new ArrayList<>();
+        return nfsfwThreadPool;
-        int index = 0;
-        subAlbums.add(new Pair(this.url.toExternalForm(), ""));
-        while (!subAlbums.isEmpty()) {
-            if (isStopped()) {
-                break;
-            }
-            Pair nextAlbum = subAlbums.remove(0);
-            String nextURL = nextAlbum.first;
-            String nextSubalbum = nextAlbum.second;
-            sendUpdate(STATUS.LOADING_RESOURCE, nextURL);
-            LOGGER.info("    Retrieving " + nextURL);
-            if (albumDoc == null) {
-                albumDoc = Http.url(nextURL).get();
-            }
-            // Subalbums
-            for (Element suba : albumDoc.select("td.IMG > a")) {
-                if (isStopped() || isThisATest()) {
-                    break;
-                }
-                String subURL = "http://nfsfw.com" + suba.attr("href");
-                String subdir = subURL;
-                while (subdir.endsWith("/")) {
-                    subdir = subdir.substring(0, subdir.length() - 1);
-                }
-                subdir = subdir.substring(subdir.lastIndexOf("/") + 1);
-                subAlbums.add(new Pair(subURL, subdir));
-            }
-            // Images
-            for (Element thumb : albumDoc.select("td.giItemCell > div > a")) {
-                if (isStopped()) {
-                    break;
-                }
-                String imagePage = "http://nfsfw.com" + thumb.attr("href");
-                try {
-                    NfsfwImageThread t = new NfsfwImageThread(new URL(imagePage), nextSubalbum, ++index);
-                    nfsfwThreadPool.addThread(t);
-                    if (isThisATest()) {
-                        break;
-                    }
-                } catch (MalformedURLException mue) {
-                    LOGGER.warn("Invalid URL: " + imagePage);
-                }
-            }
-            if (isThisATest()) {
-                break;
-            }
-            // Get next page
-            for (Element a : albumDoc.select("a.next")) {
-                subAlbums.add(0, new Pair("http://nfsfw.com" + a.attr("href"), ""));
-                break;
-            }
-            // Insert next page at the top
-            albumDoc = null;
-            // Wait
-            try {
-                Thread.sleep(1000);
-            } catch (InterruptedException e) {
-                LOGGER.error("Interrupted while waiting to load next page", e);
-                throw new IOException(e);
-            }
-        }
-        nfsfwThreadPool.waitForThreads();
-        waitForThreads();
     }
 
-    public boolean canRip(URL url) {
+    @Override
-        return url.getHost().endsWith(DOMAIN);
+    public boolean hasQueueSupport() {
+        return true;
+    }
+
+    @Override
+    public boolean pageContainsAlbums(URL url) {
+        List<String> imageURLs = getImagePageURLs(fstPage);
+        List<String> subalbumURLs = getSubalbumURLs(fstPage);
+        return imageURLs.isEmpty() && !subalbumURLs.isEmpty();
+    }
+
+    @Override
+    public List<String> getAlbumsToQueue(Document doc) {
+        return getSubalbumURLs(doc);
+    }
+
+    // helper methods
+
+    private List<String> getImagePageURLs(Document page){
+        // get image pages
+        // NOTE: It might be possible to get the (non-thumbnail) image URL
+        // without going to its page first as there seems to be a pattern
+        // between the thumb and actual image URLs, but that is outside the
+        // scope of the current issue being solved.
+        List<String> imagePageURLs = new ArrayList<>();
+        for (Element thumb : page.select("td.giItemCell > div > a")) {
+            String imagePage = "http://nfsfw.com" + thumb.attr("href");
+            imagePageURLs.add(imagePage);
+        }
+        return imagePageURLs;
+    }
+
+    private List<String> getSubalbumURLs(Document page){
+        // Check if sub-albums are present on this page
+        List<String> subalbumURLs = new ArrayList<>();
+        for (Element suba : page.select("td.IMG > a")) {
+            String subURL = "http://nfsfw.com" + suba.attr("href");
+            subalbumURLs.add(subURL);
+        }
+        return subalbumURLs;
     }
 
     /**
@@ -175,23 +226,10 @@ public class NfsfwRipper extends AlbumRipper {
                 if (file.startsWith("/")) {
                     file = "http://nfsfw.com" + file;
                 }
-                String prefix = "";
+                addURLToDownload(new URL(file), getPrefix(index), this.subdir);
-                if (Utils.getConfigBoolean("download.save_order", true)) {
-                    prefix = String.format("%03d_", index);
-                }
-                addURLToDownload(new URL(file), prefix, this.subdir);
             } catch (IOException e) {
                 LOGGER.error("[!] Exception while loading/parsing " + this.url, e);
             }
         }
     }
-
-    private class Pair {
-        String first;
-        String second;
-        Pair(String first, String second) {
-            this.first = first;
-            this.second = second;
-        }
-    }
 }

src/main/java/com/rarchives/ripme/ripper/rippers/PhotobucketRipper.java

@@ -10,23 +10,65 @@ import java.util.regex.Pattern;
 
 import org.json.JSONArray;
 import org.json.JSONObject;
-import org.jsoup.Connection.Response;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
 
-import com.rarchives.ripme.ripper.AlbumRipper;
+import com.rarchives.ripme.ripper.AbstractHTMLRipper;
 import com.rarchives.ripme.utils.Http;
 
-public class PhotobucketRipper extends AlbumRipper {
+// TODO: Probably want to add queue support for cases like this:
+// http://s732.photobucket.com/user/doublesix66/library/WARZONE?sort=3&page=1
+public class PhotobucketRipper extends AbstractHTMLRipper {
 
     private static final String DOMAIN = "photobucket.com",
                                 HOST   = "photobucket";
+    private static final int ITEMS_PER_PAGE = 24;
+    private static final int WAIT_BEFORE_NEXT_PAGE = 2000;
 
-    private Response pageResponse = null;
+    private final class AlbumMetadata {
+        private final String url;
+        private final String location;
+        private final int sortOrder;
+        private int currPage = 1;
+        private int numPages;
+
+        private AlbumMetadata(JSONObject data) {
+            this.url = data.getString("url");
+            this.location = data.getString("location")
+                                .replace(" ", "_");
+            this.sortOrder = data.getInt("sortOrder");
+        }
+
+        private String getCurrPageURL(){
+            return url + String.format("?sort=%d&page=%d",
+                                       sortOrder, currPage);
+        }
+    }
+
+    private final Pattern collDataPattern;
+    private final Pattern pbURLPattern;
+
+    // all albums including sub-albums to rip
+    private List<AlbumMetadata> albums;
+    // the album currently being ripped
+    private AlbumMetadata currAlbum;
+    // a new index per album downloaded
+    private int index = 0;
 
     public PhotobucketRipper(URL url) throws IOException {
         super(url);
+        this.collDataPattern = Pattern.compile(
+                "^.*collectionData: (\\{.*}).*$", Pattern.DOTALL
+        );
+        this.pbURLPattern = Pattern.compile(
+                "^https?://([a-zA-Z0-9]+)\\.photobucket\\.com/user/" +
+                "([a-zA-Z0-9_\\-]+)/library/([^?]*).*$"
+        );
+    }
+
+    @Override
+    protected String getDomain() {
+        return DOMAIN;
     }
 
     @Override
@@ -34,45 +76,35 @@ public class PhotobucketRipper extends AlbumRipper {
         return HOST;
     }
 
+    @Override
     public URL sanitizeURL(URL url) throws MalformedURLException {
         LOGGER.info(url);
         String u = url.toExternalForm();
         if (u.contains("?")) {
+            // strip options from URL
             u = u.substring(0, u.indexOf("?"));
+        }
+        if (!u.endsWith("/")) {
+            // append trailing slash
+            u = u + "/";
+        }
         return new URL(u);
     }
-        else {
-            return url;
-        }
-    }
-
-    public String getAlbumTitle(URL url) throws MalformedURLException {
-        try {
-            // Attempt to use album title as GID
-            if (pageResponse == null) {
-                pageResponse = Http.url(url).response();
-            }
-            Document albumDoc = pageResponse.parse();
-            Elements els = albumDoc.select("div.libraryTitle > h1");
-            if (els.isEmpty()) {
-                throw new IOException("Could not find libraryTitle at " + url);
-            }
-            return els.get(0).text();
-        } catch (IOException e) {
-            // Fall back to default album naming convention
-        }
-        return super.getAlbumTitle(url);
-    }
 
     @Override
     public String getGID(URL url) throws MalformedURLException {
-        Pattern p; Matcher m;
+        Matcher m;
+
+        URL sanitized = sanitizeURL(url);
 
         // http://s844.photobucket.com/user/SpazzySpizzy/library/Lady%20Gaga?sort=3&page=1
-        p = Pattern.compile("^https?://[a-zA-Z0-9]+\\.photobucket\\.com/user/([a-zA-Z0-9_\\-]+)/library.*$");
+        m = pbURLPattern.matcher(sanitized.toExternalForm());
-        m = p.matcher(url.toExternalForm());
         if (m.matches()) {
-            return m.group(1);
+            // the username is not really a unique GID, because the same user
+            // can have multiple albums, but on the other hand, using HOST_GID
+            // as save directory means we can group ripped albums of the same
+            // user.
+            return m.group(2);
         }
 
         throw new MalformedURLException(
@@ -81,134 +113,174 @@ public class PhotobucketRipper extends AlbumRipper {
                         + " Got: " + url);
     }
 
+
+
+    // Page iteration
+
+
+
     @Override
-    public void rip() throws IOException {
+    protected Document getFirstPage() throws IOException {
-        List<String> subalbums = ripAlbumAndGetSubalbums(this.url.toExternalForm());
+        if (this.currAlbum == null) {
-
+            this.albums = getAlbumMetadata(this.url.toExternalForm());
-        List<String> subsToRip = new ArrayList<>(),
+            LOGGER.info("Detected " + albums.size() + " albums in total");
-                    rippedSubs = new ArrayList<>();
+        }
-
+        this.currAlbum = this.albums.remove(0);
-        for (String sub : subalbums) {
+        // NOTE: Why not just get media count in the metadata json?
-            subsToRip.add(sub);
+        //
+        // Because that data might not reflect what the user sees on the page
+        // and can lead to iterating more pages than there actually are.
+        //
+        // An example:
+        // Metadata JSON -> AlbumStats: 146 images + 0 videos -> 146 items/7 pages
+        // http://s1255.photobucket.com/api/user/mimajki/album/Movie%20gifs/get?subAlbums=48&json=1
+        // Actual item count when looking at the album url: 131 items/6 pages
+        // http://s1255.photobucket.com/user/mimajki/library/Movie%20gifs?sort=6&page=1
+        Document page = Http.url(currAlbum.getCurrPageURL()).get();
+        JSONObject collectionData = getCollectionData(page);
+        int totalNumItems = collectionData.getInt("total");
+        this.currAlbum.numPages = (int) Math.ceil(
+                (double)totalNumItems / (double) ITEMS_PER_PAGE);
+        this.index = 0;
+        return page;
     }
 
-        while (!subsToRip.isEmpty() && !isStopped()) {
+    @Override
+    public Document getNextPage(Document page) throws IOException {
+        currAlbum.currPage++;
+        boolean endOfAlbum = currAlbum.currPage > currAlbum.numPages;
+        boolean noMoreSubalbums = albums.isEmpty();
+        if (endOfAlbum && noMoreSubalbums){
+            throw new IOException("No more pages");
+        }
         try {
-                Thread.sleep(1000);
+            Thread.sleep(WAIT_BEFORE_NEXT_PAGE);
         } catch (InterruptedException e) {
-                break;
+            LOGGER.info("Interrupted while waiting before getting next page");
         }
-            String nextSub = subsToRip.remove(0);
+        if (endOfAlbum){
-            rippedSubs.add(nextSub);
+            LOGGER.info("Turning to next album " + albums.get(0).url);
-            LOGGER.info("Attempting to rip next subalbum: " + nextSub);
+            return getFirstPage();
-            try {
+        } else {
-                pageResponse = null;
+            LOGGER.info("Turning to page " + currAlbum.currPage +
-                subalbums = ripAlbumAndGetSubalbums(nextSub);
+                    " of album " + currAlbum.url);
-            } catch (IOException e) {
+            return Http.url(currAlbum.getCurrPageURL()).get();
-                LOGGER.error("Error while ripping " + nextSub, e);
-                break;
         }
-            for (String subalbum : subalbums) {
-                if (!subsToRip.contains(subalbum) && !rippedSubs.contains(subalbum)) {
-                    subsToRip.add(subalbum);
-                }
-            }
-        }
-        waitForThreads();
     }
 
-    private List<String> ripAlbumAndGetSubalbums(String theUrl) throws IOException {
-        int filesIndex = 0,
-            filesTotal = 0,
-            pageIndex = 0;
-        String currentAlbumPath = null,
-               url = null;
 
-        while (pageIndex == 0 || filesIndex < filesTotal) {
+
-            if (isStopped()) {
+    // Media parsing
-                break;
+
+
+
+    @Override
+    protected List<String> getURLsFromPage(Document page) {
+        JSONObject collectionData = getCollectionData(page);
+        if (collectionData == null) {
+            LOGGER.error("Unable to find JSON data at URL: " + page.location());
+            return null;
+        } else {
+            return getImageURLs(collectionData);
         }
-            pageIndex++;
-            if (pageIndex > 1 || pageResponse == null) {
-                url = theUrl + String.format("?sort=3&page=%d", pageIndex);
-                LOGGER.info("    Retrieving " + url);
-                pageResponse = Http.url(url).response();
     }
-            Document albumDoc = pageResponse.parse();
+
-            // Retrieve JSON from request
+    private JSONObject getCollectionData(Document page){
-            String jsonString = null;
+        // Retrieve JSON from a script tag in the returned document
-            for (Element script : albumDoc.select("script[type=text/javascript]")) {
+        for (Element script : page.select("script[type=text/javascript]")) {
             String data = script.data();
             // Ensure this chunk of javascript contains the album info
-                if (!data.contains("libraryAlbumsPageCollectionData")) {
+            if (data.contains("libraryAlbumsPageCollectionData")) {
-                    continue;
+                Matcher m = collDataPattern.matcher(data);
-                }
-                // Grab the JSON
-                Pattern p; Matcher m;
-                p = Pattern.compile("^.*collectionData: (\\{.*}).*$", Pattern.DOTALL);
-                m = p.matcher(data);
                 if (m.matches()) {
-                    jsonString = m.group(1);
+                    // Grab the JSON
-                    break;
+                    return new JSONObject(m.group(1));
                 }
             }
-            if (jsonString == null) {
-                LOGGER.error("Unable to find JSON data at URL: " + url);
-                break;
         }
-            JSONObject json = new JSONObject(jsonString);
+        return  null;
+    }
+
+    private List<String> getImageURLs(JSONObject json){
+        List<String> results = new ArrayList<>();
         JSONObject items = json.getJSONObject("items");
         JSONArray objects = items.getJSONArray("objects");
-            filesTotal = items.getInt("total");
-            currentAlbumPath = json.getString("currentAlbumPath");
         for (int i = 0; i < objects.length(); i++) {
             JSONObject object = objects.getJSONObject(i);
-                String image = object.getString("fullsizeUrl");
+            String imgURL = object.getString("fullsizeUrl");
-                filesIndex += 1;
+            results.add(imgURL);
-                addURLToDownload(new URL(image),
-                        "",
-                        object.getString("location").replaceAll(" ", "_"),
-                        albumDoc.location(),
-                        pageResponse.cookies());
-            }
-        }
-        // Get subalbums
-        if (url != null) {
-            return getSubAlbums(url, currentAlbumPath);
-        } else {
-            return new ArrayList<>();
         }
+        return results;
     }
 
-    private List<String> getSubAlbums(String url, String currentAlbumPath) {
+    @Override
-        List<String> result = new ArrayList<>();
+    protected void downloadURL(URL url, int index) {
-        String subdomain = url.substring(url.indexOf("://")+3);
+        addURLToDownload(url, getPrefix(++this.index), currAlbum.location);
-        subdomain = subdomain.substring(0, subdomain.indexOf("."));
-        String apiUrl = "http://" + subdomain + ".photobucket.com/component/Albums-SubalbumList"
-                + "?deferCollapsed=true"
-                + "&albumPath=" + currentAlbumPath // %2Falbums%2Fab10%2FSpazzySpizzy"
-                + "&json=1";
-        try {
-            LOGGER.info("Loading " + apiUrl);
-            JSONObject json = Http.url(apiUrl).getJSON();
-            JSONArray subalbums = json.getJSONObject("body").getJSONArray("subAlbums");
-            for (int i = 0; i < subalbums.length(); i++) {
-                String suburl =
-                        "http://"
-                        + subdomain
-                        + ".photobucket.com"
-                        + subalbums.getJSONObject(i).getString("path");
-                suburl = suburl.replace(" ", "%20");
-                result.add(suburl);
-            }
-        } catch (IOException e) {
-            LOGGER.error("Failed to get subalbums from " + apiUrl, e);
-        }
-        return result;
     }
 
-    public boolean canRip(URL url) {
+
-        return url.getHost().endsWith(DOMAIN);
+
+    // helper methods (for album metadata retrieval)
+
+
+
+    private List<AlbumMetadata> getAlbumMetadata(String albumURL)
+            throws IOException {
+        JSONObject data = getAlbumMetadataJSON(albumURL);
+        List<AlbumMetadata> metadata = new ArrayList<>();
+        metadata.add(new AlbumMetadata(data));
+        if (!data.getString("location").equals("")) {
+            // if the location were to equal "", then we are at the profile
+            // page of a user. Ripping all sub-albums here would mean ripping
+            // all albums of a user (Not supported, only rip items in a users
+            // personal bucket).
+            for (JSONObject sub : getSubAlbumJSONs(data)){
+                metadata.add(new AlbumMetadata(sub));
+            }
+        }
+        LOGGER.info("Succesfully retrieved and parsed metadata");
+        return metadata;
     }
 
+    private JSONObject getAlbumMetadataJSON(String albumURL)
+            throws IOException {
+        String subdomain, user, albumTitle;
+        Matcher m = pbURLPattern.matcher(albumURL);
+        if (!m.matches()){
+            throw new MalformedURLException("invalid URL " + albumURL);
+        }
+        subdomain = m.group(1);
+        user = m.group(2);
+        albumTitle = m.group(3);
+        if (albumTitle.endsWith("/")){
+            albumTitle = albumTitle.substring(0, albumTitle.length() - 1);
+        }
+        String apiURL = String.format("http://%s.photobucket.com/api/user/" +
+                        "%s/album/%s/get?subAlbums=%d&json=1",
+                subdomain, user, albumTitle, ITEMS_PER_PAGE);
+        LOGGER.info("Loading " + apiURL);
+        JSONObject data = Http.url(apiURL).getJSON().getJSONObject("data");
+        if (data.has("subAlbums")) {
+            int count = data.getInt("subAlbumCount");
+            if (count > ITEMS_PER_PAGE) {
+                apiURL = String.format("http://%s.photobucket.com/api/user/" +
+                                "%s/album/%s/get?subAlbums=%d&json=1",
+                        subdomain, user, albumTitle, count);
+                data = Http.url(apiURL).getJSON().getJSONObject("data");
+            }
+        }
+        return data;
+    }
+
+    private List<JSONObject> getSubAlbumJSONs(JSONObject data) {
+        List<JSONObject> subalbumJSONs = new ArrayList<>();
+        if (data.has("subAlbums")) {
+            JSONArray subalbums = data.getJSONArray("subAlbums");
+            for (int idx = 0; idx < subalbums.length(); idx++) {
+                JSONObject subalbumJSON = subalbums.getJSONObject(idx);
+                subalbumJSONs.add(subalbumJSON);
+            }
+        }
+        return subalbumJSONs;
+    }
 }

src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java

@@ -4,9 +4,12 @@ import java.io.File;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.rarchives.ripme.ripper.AbstractHTMLRipper;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
@@ -17,7 +20,7 @@ import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
 import com.rarchives.ripme.utils.Http;
 import com.rarchives.ripme.utils.Utils;
 
-public class PornhubRipper extends AlbumRipper {
+public class PornhubRipper extends AbstractHTMLRipper {
     // All sleep times are in milliseconds
     private static final int IMAGE_SLEEP_TIME    = 1000;
 
@@ -26,9 +29,6 @@ public class PornhubRipper extends AlbumRipper {
     // Thread pool for finding direct image links from "image" pages (html)
     private DownloadThreadPool pornhubThreadPool = new DownloadThreadPool("pornhub");
 
-    // Current HTML document
-    private Document albumDoc = null;
-
     public PornhubRipper(URL url) throws IOException {
         super(url);
     }
@@ -38,25 +38,63 @@ public class PornhubRipper extends AlbumRipper {
         return HOST;
     }
 
-    public URL sanitizeURL(URL url) throws MalformedURLException {
+    @Override
-        return url;
+    protected String getDomain() {
+        return DOMAIN;
     }
 
-    public String getAlbumTitle(URL url) throws MalformedURLException {
+    @Override
+    protected Document getFirstPage() throws IOException {
+        return Http.url(url).referrer(url).get();
+    }
+
+    @Override
+    public Document getNextPage(Document page) throws IOException {
+        Elements nextPageLink = page.select("li.page_next > a");
+        if (nextPageLink.isEmpty()){
+            throw new IOException("No more pages");
+        } else {
+            URL nextURL = new URL(this.url, nextPageLink.first().attr("href"));
+            return Http.url(nextURL).get();
+        }
+    }
+
+    @Override
+    protected List<String> getURLsFromPage(Document page) {
+        List<String> pageURLs = new ArrayList<>();
+        // Find thumbnails
+        Elements thumbs = page.select(".photoBlockBox li");
+        // Iterate over thumbnail images on page
+        for (Element thumb : thumbs) {
+            String imagePage = thumb.select(".photoAlbumListBlock > a")
+                    .first().attr("href");
+            String fullURL = "https://pornhub.com" + imagePage;
+            pageURLs.add(fullURL);
+        }
+        return pageURLs;
+    }
+
+    @Override
+    protected void downloadURL(URL url, int index) {
+        PornhubImageThread t = new PornhubImageThread(url, index, this.workingDir);
+        pornhubThreadPool.addThread(t);
         try {
-            // Attempt to use album title as GID
+            Thread.sleep(IMAGE_SLEEP_TIME);
-            if (albumDoc == null) {
+        } catch (InterruptedException e) {
-                LOGGER.info("    Retrieving " + url.toExternalForm());
+            LOGGER.warn("Interrupted while waiting to load next image", e);
-                sendUpdate(STATUS.LOADING_RESOURCE, url.toString());
-                albumDoc = Http.url(url).get();
         }
-            Elements elems = albumDoc.select(".photoAlbumTitleV2");
-            return HOST + "_" + elems.get(0).text();
-        } catch (Exception e) {
-            // Fall back to default album naming convention
-            LOGGER.warn("Failed to get album title from " + url, e);
     }
-        return super.getAlbumTitle(url);
+
+    public URL sanitizeURL(URL url) throws MalformedURLException {
+        // always start on the first page of an album
+        // (strip the options after the '?')
+        String u = url.toExternalForm();
+        if (u.contains("?")) {
+            u = u.substring(0, u.indexOf("?"));
+            return new URL(u);
+        } else {
+            return url;
+        }
     }
 
     @Override
@@ -64,7 +102,7 @@ public class PornhubRipper extends AlbumRipper {
         Pattern p;
         Matcher m;
 
-        p = Pattern.compile("^.*pornhub\\.com/album/([0-9]+)$");
+        p = Pattern.compile("^.*pornhub\\.com/album/([0-9]+).*$");
         m = p.matcher(url.toExternalForm());
         if (m.matches()) {
             return m.group(1);
@@ -77,48 +115,8 @@ public class PornhubRipper extends AlbumRipper {
     }
 
     @Override
-    public void rip() throws IOException {
+    public DownloadThreadPool getThreadPool(){
-        int index = 0;
+        return pornhubThreadPool;
-        String nextUrl = this.url.toExternalForm();
-
-        if (albumDoc == null) {
-            LOGGER.info("    Retrieving album page " + nextUrl);
-            sendUpdate(STATUS.LOADING_RESOURCE, nextUrl);
-            albumDoc = Http.url(nextUrl)
-                           .referrer(this.url)
-                           .get();
-        }
-
-        // Find thumbnails
-        Elements thumbs = albumDoc.select(".photoBlockBox li");
-        if (thumbs.isEmpty()) {
-            LOGGER.debug("albumDoc: " + albumDoc);
-            LOGGER.debug("No images found at " + nextUrl);
-            return;
-        }
-
-        // Iterate over images on page
-        for (Element thumb : thumbs) {
-            if (isStopped()) {
-                break;
-            }
-            index++;
-            String imagePageUrl = thumb.select(".photoAlbumListBlock > a").first().attr("href");
-            URL imagePage = new URL(url, imagePageUrl);
-            PornhubImageThread t = new PornhubImageThread(imagePage, index, this.workingDir);
-            pornhubThreadPool.addThread(t);
-            if (isThisATest()) {
-                break;
-            }
-            try {
-                Thread.sleep(IMAGE_SLEEP_TIME);
-            } catch (InterruptedException e) {
-                LOGGER.warn("Interrupted while waiting to load next image", e);
-            }
-        }
-
-        pornhubThreadPool.waitForThreads();
-        waitForThreads();
     }
 
     public boolean canRip(URL url) {

src/main/java/com/rarchives/ripme/ripper/rippers/TeenplanetRipper.java

@@ -3,51 +3,66 @@ package com.rarchives.ripme.ripper.rippers;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.rarchives.ripme.ripper.AbstractHTMLRipper;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
 
-import com.rarchives.ripme.ripper.AlbumRipper;
-import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
 import com.rarchives.ripme.utils.Http;
 import com.rarchives.ripme.utils.Utils;
 
-public class TeenplanetRipper extends AlbumRipper {
+public class TeenplanetRipper extends AbstractHTMLRipper {
 
     private static final String DOMAIN = "teenplanet.org",
                                 HOST   = "teenplanet";
 
-    private Document albumDoc = null;
-
     public TeenplanetRipper(URL url) throws IOException {
         super(url);
     }
 
+    @Override
+    protected String getDomain() {
+        return DOMAIN;
+    }
+
     @Override
     public String getHost() {
         return HOST;
     }
 
-    public URL sanitizeURL(URL url) throws MalformedURLException {
+    @Override
-        return url;
+    protected Document getFirstPage() throws IOException {
+        return Http.url(url).get();
     }
 
-    public String getAlbumTitle(URL url) throws MalformedURLException {
+    @Override
-        try {
+    protected List<String> getURLsFromPage(Document page) {
-            // Attempt to use album title as GID
+        List<String> imageURLs = new ArrayList<>();
-            if (albumDoc == null) {
+        for (Element thumb : page.select("#galleryImages > a > img")) {
-                albumDoc = Http.url(url).get();
+            if (!thumb.hasAttr("src")) {
+                continue;
             }
-            Elements elems = albumDoc.select("div.header > h2");
+            String imageURL = thumb.attr("src");
-            return HOST + "_" + elems.get(0).text();
+            imageURL = imageURL.replace(
-        } catch (Exception e) {
+                    "/thumbs/",
-            // Fall back to default album naming convention
+                    "/");
-            e.printStackTrace();
+            imageURLs.add(imageURL);
         }
-        return super.getAlbumTitle(url);
+        System.out.println("Found" + imageURLs.size() + " image urls");
+        return imageURLs;
+    }
+
+    @Override
+    protected void downloadURL(URL url, int index) {
+        String prefix = "";
+        if (Utils.getConfigBoolean("download.save_order", true)) {
+            prefix = String.format("%03d_", index);
+        }
+        addURLToDownload(url, prefix);
     }
 
     @Override
@@ -65,38 +80,4 @@ public class TeenplanetRipper extends AlbumRipper {
                         + "teenplanet.org/galleries/....html"
                         + " Got: " + url);
     }
-
-    @Override
-    public void rip() throws IOException {
-        int index = 0;
-        LOGGER.info("Retrieving " + this.url);
-        sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
-        if (albumDoc == null) {
-            albumDoc = Http.url(url).get();
-        }
-        for (Element thumb : albumDoc.select("#galleryImages > a > img")) {
-            if (!thumb.hasAttr("src")) {
-                continue;
-            }
-            String image = thumb.attr("src");
-            image = image.replace(
-                    "/thumbs/",
-                    "/");
-            index += 1;
-            String prefix = "";
-            if (Utils.getConfigBoolean("download.save_order", true)) {
-                prefix = String.format("%03d_", index);
-            }
-            addURLToDownload(new URL(image), prefix);
-            if (isThisATest()) {
-                break;
-            }
-        }
-        waitForThreads();
-    }
-
-    public boolean canRip(URL url) {
-        return url.getHost().endsWith(DOMAIN);
-    }
-
 }

src/test/java/com/rarchives/ripme/tst/ripper/rippers/NewsfilterRipperTest.java

@@ -6,5 +6,9 @@ import java.net.URL;
 import com.rarchives.ripme.ripper.rippers.NewsfilterRipper;
 
 public class NewsfilterRipperTest extends RippersTest {
-    // TODO add a test
+
+    public void testNewsfilterRip() throws IOException {
+        NewsfilterRipper ripper = new NewsfilterRipper(new URL("http://newsfilter.org/gallery/he-doubted-she-would-fuck-on-cam-happy-to-be-proven-wrong-216799"));
+        testRipper(ripper);
+    }
 }

src/test/java/com/rarchives/ripme/tst/ripper/rippers/NfsfwRipperTest.java

@@ -6,11 +6,21 @@ import java.net.URL;
 import com.rarchives.ripme.ripper.rippers.NfsfwRipper;
 
 public class NfsfwRipperTest extends RippersTest {
-    /*
     // https://github.com/RipMeApp/ripme/issues/291 -- nfsfw "account suspended" error; disabled flaky test in CI
-    public void testNfsfwRip() throws IOException {
+    /*public void testNfsfwRip() throws IOException {
         NfsfwRipper ripper = new NfsfwRipper(new URL("http://nfsfw.com/gallery/v/Kitten/"));
         testRipper(ripper);
+    }*/
+
+    public void testGetGID() throws IOException {
+        URL url = new URL("http://nfsfw.com/gallery/v/Kitten/");
+        NfsfwRipper ripper = new NfsfwRipper(url);
+        assertEquals("Kitten", ripper.getGID(url));
+        url = new URL("http://nfsfw.com/gallery/v/Kitten");
+        assertEquals("Kitten", ripper.getGID(url));
+        url = new URL("http://nfsfw.com/gallery/v/Kitten/gif_001/");
+        assertEquals("Kitten__gif_001", ripper.getGID(url));
+        url = new URL("http://nfsfw.com/gallery/v/Kitten/gif_001/");
+        assertEquals("Kitten__gif_001", ripper.getGID(url));
     }
-    */
 }

src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java

@@ -15,7 +15,51 @@ public class PhotobucketRipperTest extends RippersTest {
         deleteDir(ripper.getWorkingDir());
     }
     */
+
+    /*
+    // new test, still commented out because of the issue above,
+    // since this test also involves network IO.
+    public void testGetNextPage() throws IOException {
+        // this album should have more than enough sub-albums and pages
+        // to serve as a pretty good iteration test (barring server or
+        // network errors)
+        String baseURL = "http://s1255.photobucket.com/user/mimajki/library/Movie%20gifs?sort=6&page=1";
+        URL url = new URL(baseURL);
+        PhotobucketRipper ripper = new PhotobucketRipper(url);
+        org.jsoup.nodes.Document page = null;
+        try {
+            // I'm not sure it makes much sense that getFirstPage()
+            // is not public while getNextPage() is.
+            java.lang.reflect.Method method = ripper.getClass()
+                                                    .getDeclaredMethod("getFirstPage");
+            method.setAccessible(true);
+            page = (org.jsoup.nodes.Document) method.invoke(ripper);
+        } catch (Exception e){
+            e.printStackTrace();
+            fail("Calling getFirstPage() failed");
+        }
+        int numPagesRemaining = 38;
+        for (int idx = 0; idx < numPagesRemaining; idx++){
+            page = ripper.getNextPage(page);
+            System.out.println("URL: " + page.location());
+        }
+        try {
+            page = ripper.getNextPage(page);
+            fail("Get next page did not throw an exception on the last page");
+        } catch(IOException e){
+            assertEquals(e.getMessage(), "No more pages");
+        }
+    }*/
+
+    public void testGetGID() throws IOException {
+        URL url = new URL("http://s732.photobucket.com/user/doublesix66/library/Army%20Painter%20examples?sort=3&page=1");
+        PhotobucketRipper ripper = new PhotobucketRipper(url);
+        assertEquals("doublesix66", ripper.getGID(url));
+        url = new URL("http://s732.photobucket.com/user/doublesix66/library/Army%20Painter%20examples/Painting%20examples?page=1&sort=3");
+        assertEquals("doublesix66", ripper.getGID(url));
+        url = new URL("http://s844.photobucket.com/user/SpazzySpizzy/library/Album%20Covers");
+        assertEquals("SpazzySpizzy", ripper.getGID(url));
+        url = new URL("http://s844.photobucket.com/user/SpazzySpizzy/library");
+        assertEquals("SpazzySpizzy", ripper.getGID(url));
+    }
 }
-
-
-

src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java

@@ -4,6 +4,8 @@ import java.io.IOException;
 import java.net.URL;
 
 import com.rarchives.ripme.ripper.rippers.PornhubRipper;
+import com.rarchives.ripme.utils.Http;
+import org.jsoup.nodes.Document;
 
 public class PornhubRipperTest extends RippersTest {
     public void testPornhubRip() throws IOException {
@@ -12,8 +14,28 @@ public class PornhubRipperTest extends RippersTest {
     }
 
     public void testGetGID() throws IOException {
-        URL url = new URL("https://www.pornhub.com/album/15680522");
+        URL url = new URL("https://www.pornhub.com/album/15680522?page=2");
         PornhubRipper ripper = new PornhubRipper(url);
         assertEquals("15680522", ripper.getGID(url));
+        url = new URL("https://www.pornhub.com/album/15680522");
+        assertEquals("15680522", ripper.getGID(url));
+    }
+
+    // alternate album, with only 2 pages: https://www.pornhub.com/album/4771891
+    public void testGetNextPage() throws IOException {
+        String baseURL = "https://www.pornhub.com/album/15680522";
+        PornhubRipper ripper = new PornhubRipper(new URL(baseURL));
+        Document page = Http.url(baseURL).get();
+        int numPagesRemaining = 4;
+        for (int idx = 0; idx < numPagesRemaining; idx++){
+            page = ripper.getNextPage(page);
+            assertEquals(baseURL + "?page=" + (idx + 2), page.location());
+        }
+        try {
+            page = ripper.getNextPage(page);
+            fail("Get next page did not throw an exception on the last page");
+        } catch(IOException e){
+            assertEquals(e.getMessage(), "No more pages");
+        }
     }
 }