From 47cab788435793ff3ad59ce9d852000a023823bb Mon Sep 17 00:00:00 2001 From: "Mateus B. Cassiano" Date: Mon, 16 Jul 2018 02:08:03 -0400 Subject: [PATCH 1/2] ArtStationRipper: fix ripping from user pages with more than 50 projects --- .../ripper/rippers/ArtStationRipper.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java index aeb1991d..611d9be6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java @@ -21,6 +21,7 @@ public class ArtStationRipper extends AbstractJSONRipper { private ParsedURL albumURL; private String projectName; private Integer projectIndex; + private Integer projectPageNumber; public ArtStationRipper(URL url) throws IOException { super(url); @@ -95,14 +96,25 @@ public class ArtStationRipper extends AbstractJSONRipper { @Override protected JSONObject getNextPage(JSONObject doc) throws IOException { if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) { - // Initialize the index if it hasn't been initialized already - if (projectIndex == null) { - projectIndex = 1; + // Initialize the page number if it hasn't been initialized already + if (projectPageNumber == null) { + projectPageNumber = 1; } - JSONObject albumContent = Http.url(albumURL.getLocation()).getJSON(); + // Each page holds a maximum of 50 projects. Initialize the index if it hasn't + // been initialized already or increment page number and reset the index if all + // projects of the current page were already processed + if (projectIndex == null) { + projectIndex = 0; + } else if (projectIndex > 49) { + projectPageNumber++; + projectIndex = 0; + } - if (albumContent.getInt("total_count") > projectIndex) { + Integer currentProject = ((projectPageNumber - 1) * 50) + (projectIndex + 1); + JSONObject albumContent = Http.url(albumURL.getLocation() + "?page=" + projectPageNumber).getJSON(); + + if (albumContent.getInt("total_count") > currentProject) { // Get JSON of the next project and return it JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(projectIndex); ParsedURL projectURL = parseURL(new URL(projectInfo.getString("permalink"))); From 9bb28cf4c7f40718835f144902892450f27be0a0 Mon Sep 17 00:00:00 2001 From: "Mateus B. Cassiano" Date: Mon, 16 Jul 2018 02:09:06 -0400 Subject: [PATCH 2/2] ArtStationRipper: update unit test --- .../rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java index 381c007f..fe72c046 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java @@ -23,6 +23,7 @@ public class ArtStationRipperTest extends RippersTest { public void testArtStationUserProfiles() throws IOException { List contentURLs = new ArrayList<>(); contentURLs.add(new URL("https://www.artstation.com/heitoramatsu")); + contentURLs.add(new URL("https://artstation.com/kuvshinov_ilya")); contentURLs.add(new URL("http://artstation.com/givemeapiggy")); for (URL url : contentURLs) { ArtStationRipper ripper = new ArtStationRipper(url);