diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 06283ebf..0c095b28 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,15 +1,15 @@ -* Ripme version: -* Java version: -* Operating system: - -* Exact URL you were trying to rip when the problem occurred: -* Please include any additional information about how to reproduce the problem: - -## Expected Behavior - -Detail the expected behavior here. - -## Actual Behavior - -Detail the actual (incorrect) behavior here. You can post log snippets or attach log files to your issue report. +* Ripme version: +* Java version: +* Operating system: + +* Exact URL you were trying to rip when the problem occurred: +* Please include any additional information about how to reproduce the problem: + +## Expected Behavior + +Detail the expected behavior here. + +## Actual Behavior + +Detail the actual (incorrect) behavior here. You can post log snippets or attach log files to your issue report. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 8810800c..56d0dd29 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,27 +1,27 @@ -# Category - -This change is exactly one of the following (please change `[ ]` to `[x]`) to indicate which: -* [ ] a bug fix (Fix #...) -* [ ] a new Ripper -* [ ] a refactoring -* [ ] a style change/fix -* [ ] a new feature - - -# Description - -Please add details about your change here. - - -# Testing - -Required verification: -* [ ] I've verified that there are no regressions in `mvn test` (there are no new failures or errors). -* [ ] I've verified that this change works as intended. - * [ ] Downloads all relevant content. - * [ ] Downloads content from multiple pages (as necessary or appropriate). - * [ ] Saves content at reasonable file names (e.g. page titles or content IDs) to help easily browse downloaded content. -* [ ] I've verified that this change did not break existing functionality (especially in the Ripper I modified). - -Optional but recommended: -* [ ] I've added a unit test to cover my change. +# Category + +This change is exactly one of the following (please change `[ ]` to `[x]`) to indicate which: +* [ ] a bug fix (Fix #...) +* [ ] a new Ripper +* [ ] a refactoring +* [ ] a style change/fix +* [ ] a new feature + + +# Description + +Please add details about your change here. + + +# Testing + +Required verification: +* [ ] I've verified that there are no regressions in `mvn test` (there are no new failures or errors). +* [ ] I've verified that this change works as intended. + * [ ] Downloads all relevant content. + * [ ] Downloads content from multiple pages (as necessary or appropriate). + * [ ] Saves content at reasonable file names (e.g. page titles or content IDs) to help easily browse downloaded content. +* [ ] I've verified that this change did not break existing functionality (especially in the Ripper I modified). + +Optional but recommended: +* [ ] I've added a unit test to cover my change. diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ArtstnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtstnRipper.java index 82b6e97c..7505c078 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ArtstnRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtstnRipper.java @@ -1,58 +1,58 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; - -import org.jsoup.Connection.Response; - -import com.rarchives.ripme.utils.Http; - -/* - * Ripper for ArtStation's short URL domain. - * Example URL: https://artstn.co/p/JlE15Z - */ - -public class ArtstnRipper extends ArtStationRipper { - public URL artStationUrl = null; - - public ArtstnRipper(URL url) throws IOException { - super(url); - } - - @Override - public boolean canRip(URL url) { - return url.getHost().endsWith("artstn.co"); - } - - @Override - public String getGID(URL url) throws MalformedURLException { - if (artStationUrl == null) { - // Run only once. - try { - artStationUrl = getFinalUrl(url); - if (artStationUrl == null) { - throw new IOException("Null url received."); - } - } catch (IOException e) { - LOGGER.error("Couldnt resolve URL.", e); - } - - } - return super.getGID(artStationUrl); - } - - public URL getFinalUrl(URL url) throws IOException { - if (url.getHost().endsWith("artstation.com")) { - return url; - } - - LOGGER.info("Checking url: " + url); - Response response = Http.url(url).connection().followRedirects(false).execute(); - if (response.statusCode() / 100 == 3 && response.hasHeader("location")) { - return getFinalUrl(new URL(response.header("location"))); - } else { - return null; - } - } -} +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; + +import org.jsoup.Connection.Response; + +import com.rarchives.ripme.utils.Http; + +/* + * Ripper for ArtStation's short URL domain. + * Example URL: https://artstn.co/p/JlE15Z + */ + +public class ArtstnRipper extends ArtStationRipper { + public URL artStationUrl = null; + + public ArtstnRipper(URL url) throws IOException { + super(url); + } + + @Override + public boolean canRip(URL url) { + return url.getHost().endsWith("artstn.co"); + } + + @Override + public String getGID(URL url) throws MalformedURLException { + if (artStationUrl == null) { + // Run only once. + try { + artStationUrl = getFinalUrl(url); + if (artStationUrl == null) { + throw new IOException("Null url received."); + } + } catch (IOException e) { + LOGGER.error("Couldnt resolve URL.", e); + } + + } + return super.getGID(artStationUrl); + } + + public URL getFinalUrl(URL url) throws IOException { + if (url.getHost().endsWith("artstation.com")) { + return url; + } + + LOGGER.info("Checking url: " + url); + Response response = Http.url(url).connection().followRedirects(false).execute(); + if (response.statusCode() / 100 == 3 && response.hasHeader("location")) { + return getFinalUrl(new URL(response.header("location"))); + } else { + return null; + } + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ComicextraRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ComicextraRipper.java index 3abf749a..e794e072 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ComicextraRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ComicextraRipper.java @@ -1,174 +1,174 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; - -/** - * @author Tushar - * - */ -public class ComicextraRipper extends AbstractHTMLRipper { - - private static final String FILE_NAME = "page"; - - private Pattern p1 = - Pattern.compile("https:\\/\\/www.comicextra.com\\/comic\\/([A-Za-z0-9_-]+)"); - private Pattern p2 = Pattern.compile( - "https:\\/\\/www.comicextra.com\\/([A-Za-z0-9_-]+)\\/([A-Za-z0-9_-]+)(?:\\/full)?"); - private UrlType urlType = UrlType.UNKNOWN; - private List chaptersList = null; - private int chapterIndex = -1; // index for the chaptersList, useful in getting the next page. - private int imageIndex = 0; // image index for each chapter images. - - public ComicextraRipper(URL url) throws IOException { - super(url); - } - - @Override - protected String getDomain() { - return "comicextra.com"; - } - - @Override - public String getHost() { - return "comicextra"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Matcher m1 = p1.matcher(url.toExternalForm()); - if (m1.matches()) { - // URL is of comic( https://www.comicextra.com/comic/the-punisher-frank-castle-max). - urlType = UrlType.COMIC; - return m1.group(1); - } - - Matcher m2 = p2.matcher(url.toExternalForm()); - if (m2.matches()) { - // URL is of chapter( https://www.comicextra.com/the-punisher-frank-castle-max/chapter-75). - urlType = UrlType.CHAPTER; - return m2.group(1); - } - - throw new MalformedURLException( - "Expected comicextra.com url of type: https://www.comicextra.com/comic/some-comic-name\n" - + " or https://www.comicextra.com/some-comic-name/chapter-001 got " + url - + " instead"); - } - - @Override - protected Document getFirstPage() throws IOException { - Document doc = null; - - switch (urlType) { - case COMIC: - // For COMIC type url we extract the urls of each chapters and store them in chapters. - chaptersList = new ArrayList<>(); - Document comicPage = Http.url(url).get(); - Elements elements = comicPage.select("div.episode-list a"); - for (Element e : elements) { - chaptersList.add(getCompleteChapterUrl(e.attr("abs:href"))); - } - - // Set the first chapter from the chapterList as the doc. - chapterIndex = 0; - doc = Http.url(chaptersList.get(chapterIndex)).get(); - break; - case CHAPTER: - doc = Http.url(url).get(); - break; - case UNKNOWN: - default: - throw new IOException("Unknown url type encountered."); - } - - return doc; - } - - @Override - public Document getNextPage(Document doc) throws IOException, URISyntaxException { - if (urlType == UrlType.COMIC) { - ++chapterIndex; - imageIndex = 0; // Resetting the imagesIndex so that images prefix within each chapter starts from '001_'. - if (chapterIndex < chaptersList.size()) { - return Http.url(chaptersList.get(chapterIndex)).get(); - } - } - - return super.getNextPage(doc); - } - - @Override - protected List getURLsFromPage(Document page) { - List urls = new ArrayList<>(); - - if (urlType == UrlType.COMIC || urlType == UrlType.CHAPTER) { - Elements images = page.select("img.chapter_img"); - for (Element img : images) { - urls.add(img.attr("src")); - } - } - - return urls; - } - - @Override - protected void downloadURL(URL url, int index) { - String subdirectory = getSubDirectoryName(); - String prefix = getPrefix(++imageIndex); - - addURLToDownload(url, subdirectory, null, null, prefix, FILE_NAME, null, Boolean.TRUE); - } - - /* - * This function appends /full at the end of the chapters url to get all the images for the - * chapter in the same Document. - */ - private String getCompleteChapterUrl(String chapterUrl) { - if (!chapterUrl.endsWith("/full")) { - chapterUrl = chapterUrl + "/full"; - } - return chapterUrl; - } - - /* - * This functions returns sub folder name for the current chapter. - */ - private String getSubDirectoryName() { - String subDirectory = ""; - - if (urlType == UrlType.COMIC) { - Matcher m = p2.matcher(chaptersList.get(chapterIndex)); - if (m.matches()) { - subDirectory = m.group(2); - } - } - - if (urlType == UrlType.CHAPTER) { - Matcher m = p2.matcher(url.toExternalForm()); - if (m.matches()) { - subDirectory = m.group(2); - } - } - - return subDirectory; - } - - /* - * Enum to classify different types of urls. - */ - private enum UrlType { - COMIC, CHAPTER, UNKNOWN - } -} +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +/** + * @author Tushar + * + */ +public class ComicextraRipper extends AbstractHTMLRipper { + + private static final String FILE_NAME = "page"; + + private Pattern p1 = + Pattern.compile("https:\\/\\/www.comicextra.com\\/comic\\/([A-Za-z0-9_-]+)"); + private Pattern p2 = Pattern.compile( + "https:\\/\\/www.comicextra.com\\/([A-Za-z0-9_-]+)\\/([A-Za-z0-9_-]+)(?:\\/full)?"); + private UrlType urlType = UrlType.UNKNOWN; + private List chaptersList = null; + private int chapterIndex = -1; // index for the chaptersList, useful in getting the next page. + private int imageIndex = 0; // image index for each chapter images. + + public ComicextraRipper(URL url) throws IOException { + super(url); + } + + @Override + protected String getDomain() { + return "comicextra.com"; + } + + @Override + public String getHost() { + return "comicextra"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Matcher m1 = p1.matcher(url.toExternalForm()); + if (m1.matches()) { + // URL is of comic( https://www.comicextra.com/comic/the-punisher-frank-castle-max). + urlType = UrlType.COMIC; + return m1.group(1); + } + + Matcher m2 = p2.matcher(url.toExternalForm()); + if (m2.matches()) { + // URL is of chapter( https://www.comicextra.com/the-punisher-frank-castle-max/chapter-75). + urlType = UrlType.CHAPTER; + return m2.group(1); + } + + throw new MalformedURLException( + "Expected comicextra.com url of type: https://www.comicextra.com/comic/some-comic-name\n" + + " or https://www.comicextra.com/some-comic-name/chapter-001 got " + url + + " instead"); + } + + @Override + protected Document getFirstPage() throws IOException { + Document doc = null; + + switch (urlType) { + case COMIC: + // For COMIC type url we extract the urls of each chapters and store them in chapters. + chaptersList = new ArrayList<>(); + Document comicPage = Http.url(url).get(); + Elements elements = comicPage.select("div.episode-list a"); + for (Element e : elements) { + chaptersList.add(getCompleteChapterUrl(e.attr("abs:href"))); + } + + // Set the first chapter from the chapterList as the doc. + chapterIndex = 0; + doc = Http.url(chaptersList.get(chapterIndex)).get(); + break; + case CHAPTER: + doc = Http.url(url).get(); + break; + case UNKNOWN: + default: + throw new IOException("Unknown url type encountered."); + } + + return doc; + } + + @Override + public Document getNextPage(Document doc) throws IOException, URISyntaxException { + if (urlType == UrlType.COMIC) { + ++chapterIndex; + imageIndex = 0; // Resetting the imagesIndex so that images prefix within each chapter starts from '001_'. + if (chapterIndex < chaptersList.size()) { + return Http.url(chaptersList.get(chapterIndex)).get(); + } + } + + return super.getNextPage(doc); + } + + @Override + protected List getURLsFromPage(Document page) { + List urls = new ArrayList<>(); + + if (urlType == UrlType.COMIC || urlType == UrlType.CHAPTER) { + Elements images = page.select("img.chapter_img"); + for (Element img : images) { + urls.add(img.attr("src")); + } + } + + return urls; + } + + @Override + protected void downloadURL(URL url, int index) { + String subdirectory = getSubDirectoryName(); + String prefix = getPrefix(++imageIndex); + + addURLToDownload(url, subdirectory, null, null, prefix, FILE_NAME, null, Boolean.TRUE); + } + + /* + * This function appends /full at the end of the chapters url to get all the images for the + * chapter in the same Document. + */ + private String getCompleteChapterUrl(String chapterUrl) { + if (!chapterUrl.endsWith("/full")) { + chapterUrl = chapterUrl + "/full"; + } + return chapterUrl; + } + + /* + * This functions returns sub folder name for the current chapter. + */ + private String getSubDirectoryName() { + String subDirectory = ""; + + if (urlType == UrlType.COMIC) { + Matcher m = p2.matcher(chaptersList.get(chapterIndex)); + if (m.matches()) { + subDirectory = m.group(2); + } + } + + if (urlType == UrlType.CHAPTER) { + Matcher m = p2.matcher(url.toExternalForm()); + if (m.matches()) { + subDirectory = m.group(2); + } + } + + return subDirectory; + } + + /* + * Enum to classify different types of urls. + */ + private enum UrlType { + COMIC, CHAPTER, UNKNOWN + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java index 0e0220f9..81a39823 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java @@ -1,55 +1,55 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.*; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -public class CyberdropRipper extends AbstractHTMLRipper { - - public CyberdropRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "cyberdrop"; - } - - @Override - public String getDomain() { - return "cyberdrop.me"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://cyberdrop\\.me/a/([a-zA-Z0-9]+).*?$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - throw new MalformedURLException("Expected cyberdrop.me URL format: " + - "https://cyberdrop.me/a/xxxxxxxx - got " + url + "instead"); - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } - - @Override - protected List getURLsFromPage(Document page) { - ArrayList urls = new ArrayList<>(); - for (Element element: page.getElementsByClass("image")) { - urls.add(element.attr("href")); - } - return urls; - } +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.*; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +public class CyberdropRipper extends AbstractHTMLRipper { + + public CyberdropRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "cyberdrop"; + } + + @Override + public String getDomain() { + return "cyberdrop.me"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^https?://cyberdrop\\.me/a/([a-zA-Z0-9]+).*?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected cyberdrop.me URL format: " + + "https://cyberdrop.me/a/xxxxxxxx - got " + url + "instead"); + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + + @Override + protected List getURLsFromPage(Document page) { + ArrayList urls = new ArrayList<>(); + for (Element element: page.getElementsByClass("image")) { + urls.add(element.attr("href")); + } + return urls; + } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FitnakedgirlsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FitnakedgirlsRipper.java index 3c0a68f0..51d5f15f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FitnakedgirlsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FitnakedgirlsRipper.java @@ -1,66 +1,66 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; - -public class FitnakedgirlsRipper extends AbstractHTMLRipper { - - public FitnakedgirlsRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "fitnakedgirls"; - } - - @Override - public String getDomain() { - return "fitnakedgirls.com"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p; - Matcher m; - - p = Pattern.compile("^.*fitnakedgirls\\.com/gallery/(.+)$"); - m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - - throw new MalformedURLException( - "Expected fitnakedgirls.com gallery format: " + "fitnakedgirls.com/gallery/####" + " Got: " + url); - } - - @Override - public List getURLsFromPage(Document doc) { - List imageURLs = new ArrayList<>(); - - Elements imgs = doc.select("div[class*=wp-tiles-tile-bg] > img"); - for (Element img : imgs) { - String imgSrc = img.attr("src"); - imageURLs.add(imgSrc); - } - - return imageURLs; - } - - @Override - public void downloadURL(URL url, int index) { - // Send referrer when downloading images - addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), null); - } +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; + +public class FitnakedgirlsRipper extends AbstractHTMLRipper { + + public FitnakedgirlsRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "fitnakedgirls"; + } + + @Override + public String getDomain() { + return "fitnakedgirls.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p; + Matcher m; + + p = Pattern.compile("^.*fitnakedgirls\\.com/gallery/(.+)$"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + + throw new MalformedURLException( + "Expected fitnakedgirls.com gallery format: " + "fitnakedgirls.com/gallery/####" + " Got: " + url); + } + + @Override + public List getURLsFromPage(Document doc) { + List imageURLs = new ArrayList<>(); + + Elements imgs = doc.select("div[class*=wp-tiles-tile-bg] > img"); + for (Element img : imgs) { + String imgSrc = img.attr("src"); + imageURLs.add(imgSrc); + } + + return imageURLs; + } + + @Override + public void downloadURL(URL url, int index) { + // Send referrer when downloading images + addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), null); + } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java index 614a575f..2df6ab2c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java @@ -1,293 +1,293 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.net.*; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.java_websocket.client.WebSocketClient; - -import org.apache.http.NameValuePair; -import org.apache.http.client.utils.URLEncodedUtils; -import org.java_websocket.handshake.ServerHandshake; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -import com.rarchives.ripme.ripper.AbstractJSONRipper; - -public class ScrolllerRipper extends AbstractJSONRipper { - - public ScrolllerRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "scrolller"; - } - @Override - public String getDomain() { - return "scrolller.com"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - // Typical URL is: https://scrolller.com/r/subreddit - // Parameters like "filter" and "sort" can be passed (ex: https://scrolller.com/r/subreddit?filter=xxx&sort=yyyy) - Pattern p = Pattern.compile("^https?://scrolller\\.com/r/([a-zA-Z0-9]+).*?$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - throw new MalformedURLException("Expected scrolller.com URL format: " + - "scrolller.com/r/subreddit OR scroller.com/r/subreddit?filter= - got " + url + "instead"); - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } - - - private JSONObject prepareQuery(String iterator, String gid, String sortByString) throws IOException, URISyntaxException { - - String QUERY_NOSORT = "query SubredditQuery( $url: String! $filter: SubredditPostFilter $iterator: String ) { getSubreddit(url: $url) { children( limit: 50 iterator: $iterator filter: $filter ) { iterator items { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } } } }"; - String QUERY_SORT = "subscription SubredditSubscription( $url: String! $sortBy: SubredditSortBy $timespan: SubredditTimespan $iterator: String $limit: Int $filter: SubredditPostFilter ) { fetchSubreddit( url: $url sortBy: $sortBy timespan: $timespan iterator: $iterator limit: $limit filter: $filter ) { __typename ... on Subreddit { __typename url title secondaryTitle description createdAt isNsfw subscribers isComplete itemCount videoCount pictureCount albumCount isFollowing } ... on SubredditPost { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } ... on Iterator { iterator } ... on Error { message } } }"; - - String filterString = convertFilterString(getParameter(this.url,"filter")); - - JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid)).put("sortBy", sortByString.toUpperCase()); - JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", sortByString.equals("") ? QUERY_NOSORT : QUERY_SORT); - - if (iterator != null) { - // Iterator is not present on the first page - variablesObject.put("iterator", iterator); - } - if (!filterString.equals("NOFILTER")) { - variablesObject.put("filter", filterString); - } - - return sortByString.equals("") ? getPosts(finalQueryObject) : getPostsSorted(finalQueryObject); - - } - - - public String convertFilterString(String filterParameter) { - // Converts the ?filter= parameter of the URL to one that can be used in the GraphQL query - // I could basically remove the last "s" and call toUpperCase instead of this switch statement but this looks easier to read. - switch (filterParameter.toLowerCase()) { - case "pictures": - return "PICTURE"; - case "videos": - return "VIDEO"; - case "albums": - return "ALBUM"; - case "": - return "NOFILTER"; - default: - LOGGER.error(String.format("Invalid filter %s using no filter",filterParameter)); - return ""; - } - } - - public String getParameter(URL url, String parameter) throws MalformedURLException { - // Gets passed parameters from the URL - String toReplace = String.format("https://scrolller.com/r/%s?",getGID(url)); - List args= URLEncodedUtils.parse(url.toExternalForm(), Charset.defaultCharset()); - for (NameValuePair arg:args) { - // First parameter contains part of the url so we have to remove it - // Ex: for the url https://scrolller.com/r/CatsStandingUp?filter=xxxx&sort=yyyy - // 1) arg.getName() => https://scrolller.com/r/CatsStandingUp?filter - // 2) arg.getName() => sort - - if (arg.getName().replace(toReplace,"").toLowerCase().equals((parameter))) { - return arg.getValue(); - } - } - return ""; - } - - private JSONObject getPosts(JSONObject data) { - // The actual GraphQL query call - - try { - String url = "https://api.scrolller.com/api/v2/graphql"; - - URL obj = new URI(url).toURL(); - HttpURLConnection conn = (HttpURLConnection) obj.openConnection(); - conn.setReadTimeout(5000); - conn.addRequestProperty("Accept-Language", "en-US,en;q=0.8"); - conn.addRequestProperty("User-Agent", "Mozilla"); - conn.addRequestProperty("Referer", "scrolller.com"); - - conn.setDoOutput(true); - - OutputStreamWriter w = new OutputStreamWriter(conn.getOutputStream(), "UTF-8"); - - w.write(data.toString()); - w.close(); - - BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream())); - String inputLine; - StringBuffer jsonString = new StringBuffer(); - - while ((inputLine = in.readLine()) != null) { - jsonString.append(inputLine); - } - - in.close(); - conn.disconnect(); - - return new JSONObject(jsonString.toString()); - - } catch (Exception e) { - e.printStackTrace(); - } - - return new JSONObject("{}"); - } - - private JSONObject getPostsSorted(JSONObject data) throws MalformedURLException { - - // The actual GraphQL query call (if sort parameter is present) - try { - - ArrayList postsJsonStrings = new ArrayList<>(); - - WebSocketClient wsc = new WebSocketClient(new URI("wss://api.scrolller.com/api/v2/graphql")) { - @Override - public void onOpen(ServerHandshake serverHandshake) { - // As soon as the WebSocket connects send our query - this.send(data.toString()); - } - - @Override - public void onMessage(String s) { - postsJsonStrings.add(s); - if (new JSONObject(s).getJSONObject("data").getJSONObject("fetchSubreddit").has("iterator")) { - this.close(); - } - } - - @Override - public void onClose(int i, String s, boolean b) { - } - - @Override - public void onError(Exception e) { - LOGGER.error(String.format("WebSocket error, server reported %s", e.getMessage())); - } - }; - wsc.connect(); - - while (!wsc.isClosed()) { - // Posts list is not over until the connection closes. - } - - JSONObject finalObject = new JSONObject(); - JSONArray posts = new JSONArray(); - - // Iterator is the last object in the post list, let's duplicate it in his own object for clarity. - finalObject.put("iterator", new JSONObject(postsJsonStrings.get(postsJsonStrings.size()-1))); - - for (String postString : postsJsonStrings) { - posts.put(new JSONObject(postString)); - } - finalObject.put("posts", posts); - - if (finalObject.getJSONArray("posts").length() == 1 && !finalObject.getJSONArray("posts").getJSONObject(0).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { - // Only iterator, no posts. - return null; - } - - return finalObject; - - - } catch (URISyntaxException ue) { - // Nothing to catch, it's an hardcoded URI. - } - - return null; - } - - - @Override - protected List getURLsFromJSON(JSONObject json) throws JSONException { - - boolean sortRequested = json.has("posts"); - - int bestArea = 0; - String bestUrl = ""; - List list = new ArrayList<>(); - - JSONArray itemsList = sortRequested ? json.getJSONArray("posts") : json.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").getJSONArray("items"); - - for (Object item : itemsList) { - - if (sortRequested && !((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { - continue; - } - - JSONArray sourcesTMP = sortRequested ? ((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").getJSONArray("mediaSources") : ((JSONObject) item).getJSONArray("mediaSources"); - for (Object sourceTMP : sourcesTMP) - { - int widthTMP = ((JSONObject) sourceTMP).getInt("width"); - int heightTMP = ((JSONObject) sourceTMP).getInt("height"); - int areaTMP = widthTMP * heightTMP; - - if (areaTMP > bestArea) { - bestArea = widthTMP; - bestUrl = ((JSONObject) sourceTMP).getString("url"); - } - } - list.add(bestUrl); - bestUrl = ""; - bestArea = 0; - } - - return list; - } - - @Override - protected JSONObject getFirstPage() throws IOException { - try { - return prepareQuery(null, this.getGID(url), getParameter(url,"sort")); - } catch (URISyntaxException e) { - LOGGER.error(String.format("Error obtaining first page: %s", e.getMessage())); - return null; - } - } - - @Override - public JSONObject getNextPage(JSONObject source) throws IOException { - // Every call the the API contains an "iterator" string that we need to pass to the API to get the next page - // Checking if iterator is null is not working for some reason, hence why the weird "iterator.toString().equals("null")" - - Object iterator = null; - if (source.has("iterator")) { - // Sort requested, custom JSON. - iterator = source.getJSONObject("iterator").getJSONObject("data").getJSONObject("fetchSubreddit").get("iterator"); - } else { - iterator = source.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").get("iterator"); - } - - if (!iterator.toString().equals("null")) { - // Need to change page. - try { - return prepareQuery(iterator.toString(), this.getGID(url), getParameter(url,"sort")); - } catch (URISyntaxException e) { - LOGGER.error(String.format("Error changing page: %s", e.getMessage())); - return null; - } - } else { - return null; - } - } +package com.rarchives.ripme.ripper.rippers; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.net.*; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.java_websocket.client.WebSocketClient; + +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; +import org.java_websocket.handshake.ServerHandshake; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import com.rarchives.ripme.ripper.AbstractJSONRipper; + +public class ScrolllerRipper extends AbstractJSONRipper { + + public ScrolllerRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "scrolller"; + } + @Override + public String getDomain() { + return "scrolller.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + // Typical URL is: https://scrolller.com/r/subreddit + // Parameters like "filter" and "sort" can be passed (ex: https://scrolller.com/r/subreddit?filter=xxx&sort=yyyy) + Pattern p = Pattern.compile("^https?://scrolller\\.com/r/([a-zA-Z0-9]+).*?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected scrolller.com URL format: " + + "scrolller.com/r/subreddit OR scroller.com/r/subreddit?filter= - got " + url + "instead"); + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + + + private JSONObject prepareQuery(String iterator, String gid, String sortByString) throws IOException, URISyntaxException { + + String QUERY_NOSORT = "query SubredditQuery( $url: String! $filter: SubredditPostFilter $iterator: String ) { getSubreddit(url: $url) { children( limit: 50 iterator: $iterator filter: $filter ) { iterator items { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } } } }"; + String QUERY_SORT = "subscription SubredditSubscription( $url: String! $sortBy: SubredditSortBy $timespan: SubredditTimespan $iterator: String $limit: Int $filter: SubredditPostFilter ) { fetchSubreddit( url: $url sortBy: $sortBy timespan: $timespan iterator: $iterator limit: $limit filter: $filter ) { __typename ... on Subreddit { __typename url title secondaryTitle description createdAt isNsfw subscribers isComplete itemCount videoCount pictureCount albumCount isFollowing } ... on SubredditPost { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } ... on Iterator { iterator } ... on Error { message } } }"; + + String filterString = convertFilterString(getParameter(this.url,"filter")); + + JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid)).put("sortBy", sortByString.toUpperCase()); + JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", sortByString.equals("") ? QUERY_NOSORT : QUERY_SORT); + + if (iterator != null) { + // Iterator is not present on the first page + variablesObject.put("iterator", iterator); + } + if (!filterString.equals("NOFILTER")) { + variablesObject.put("filter", filterString); + } + + return sortByString.equals("") ? getPosts(finalQueryObject) : getPostsSorted(finalQueryObject); + + } + + + public String convertFilterString(String filterParameter) { + // Converts the ?filter= parameter of the URL to one that can be used in the GraphQL query + // I could basically remove the last "s" and call toUpperCase instead of this switch statement but this looks easier to read. + switch (filterParameter.toLowerCase()) { + case "pictures": + return "PICTURE"; + case "videos": + return "VIDEO"; + case "albums": + return "ALBUM"; + case "": + return "NOFILTER"; + default: + LOGGER.error(String.format("Invalid filter %s using no filter",filterParameter)); + return ""; + } + } + + public String getParameter(URL url, String parameter) throws MalformedURLException { + // Gets passed parameters from the URL + String toReplace = String.format("https://scrolller.com/r/%s?",getGID(url)); + List args= URLEncodedUtils.parse(url.toExternalForm(), Charset.defaultCharset()); + for (NameValuePair arg:args) { + // First parameter contains part of the url so we have to remove it + // Ex: for the url https://scrolller.com/r/CatsStandingUp?filter=xxxx&sort=yyyy + // 1) arg.getName() => https://scrolller.com/r/CatsStandingUp?filter + // 2) arg.getName() => sort + + if (arg.getName().replace(toReplace,"").toLowerCase().equals((parameter))) { + return arg.getValue(); + } + } + return ""; + } + + private JSONObject getPosts(JSONObject data) { + // The actual GraphQL query call + + try { + String url = "https://api.scrolller.com/api/v2/graphql"; + + URL obj = new URI(url).toURL(); + HttpURLConnection conn = (HttpURLConnection) obj.openConnection(); + conn.setReadTimeout(5000); + conn.addRequestProperty("Accept-Language", "en-US,en;q=0.8"); + conn.addRequestProperty("User-Agent", "Mozilla"); + conn.addRequestProperty("Referer", "scrolller.com"); + + conn.setDoOutput(true); + + OutputStreamWriter w = new OutputStreamWriter(conn.getOutputStream(), "UTF-8"); + + w.write(data.toString()); + w.close(); + + BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream())); + String inputLine; + StringBuffer jsonString = new StringBuffer(); + + while ((inputLine = in.readLine()) != null) { + jsonString.append(inputLine); + } + + in.close(); + conn.disconnect(); + + return new JSONObject(jsonString.toString()); + + } catch (Exception e) { + e.printStackTrace(); + } + + return new JSONObject("{}"); + } + + private JSONObject getPostsSorted(JSONObject data) throws MalformedURLException { + + // The actual GraphQL query call (if sort parameter is present) + try { + + ArrayList postsJsonStrings = new ArrayList<>(); + + WebSocketClient wsc = new WebSocketClient(new URI("wss://api.scrolller.com/api/v2/graphql")) { + @Override + public void onOpen(ServerHandshake serverHandshake) { + // As soon as the WebSocket connects send our query + this.send(data.toString()); + } + + @Override + public void onMessage(String s) { + postsJsonStrings.add(s); + if (new JSONObject(s).getJSONObject("data").getJSONObject("fetchSubreddit").has("iterator")) { + this.close(); + } + } + + @Override + public void onClose(int i, String s, boolean b) { + } + + @Override + public void onError(Exception e) { + LOGGER.error(String.format("WebSocket error, server reported %s", e.getMessage())); + } + }; + wsc.connect(); + + while (!wsc.isClosed()) { + // Posts list is not over until the connection closes. + } + + JSONObject finalObject = new JSONObject(); + JSONArray posts = new JSONArray(); + + // Iterator is the last object in the post list, let's duplicate it in his own object for clarity. + finalObject.put("iterator", new JSONObject(postsJsonStrings.get(postsJsonStrings.size()-1))); + + for (String postString : postsJsonStrings) { + posts.put(new JSONObject(postString)); + } + finalObject.put("posts", posts); + + if (finalObject.getJSONArray("posts").length() == 1 && !finalObject.getJSONArray("posts").getJSONObject(0).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { + // Only iterator, no posts. + return null; + } + + return finalObject; + + + } catch (URISyntaxException ue) { + // Nothing to catch, it's an hardcoded URI. + } + + return null; + } + + + @Override + protected List getURLsFromJSON(JSONObject json) throws JSONException { + + boolean sortRequested = json.has("posts"); + + int bestArea = 0; + String bestUrl = ""; + List list = new ArrayList<>(); + + JSONArray itemsList = sortRequested ? json.getJSONArray("posts") : json.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").getJSONArray("items"); + + for (Object item : itemsList) { + + if (sortRequested && !((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { + continue; + } + + JSONArray sourcesTMP = sortRequested ? ((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").getJSONArray("mediaSources") : ((JSONObject) item).getJSONArray("mediaSources"); + for (Object sourceTMP : sourcesTMP) + { + int widthTMP = ((JSONObject) sourceTMP).getInt("width"); + int heightTMP = ((JSONObject) sourceTMP).getInt("height"); + int areaTMP = widthTMP * heightTMP; + + if (areaTMP > bestArea) { + bestArea = widthTMP; + bestUrl = ((JSONObject) sourceTMP).getString("url"); + } + } + list.add(bestUrl); + bestUrl = ""; + bestArea = 0; + } + + return list; + } + + @Override + protected JSONObject getFirstPage() throws IOException { + try { + return prepareQuery(null, this.getGID(url), getParameter(url,"sort")); + } catch (URISyntaxException e) { + LOGGER.error(String.format("Error obtaining first page: %s", e.getMessage())); + return null; + } + } + + @Override + public JSONObject getNextPage(JSONObject source) throws IOException { + // Every call the the API contains an "iterator" string that we need to pass to the API to get the next page + // Checking if iterator is null is not working for some reason, hence why the weird "iterator.toString().equals("null")" + + Object iterator = null; + if (source.has("iterator")) { + // Sort requested, custom JSON. + iterator = source.getJSONObject("iterator").getJSONObject("data").getJSONObject("fetchSubreddit").get("iterator"); + } else { + iterator = source.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").get("iterator"); + } + + if (!iterator.toString().equals("null")) { + // Need to change page. + try { + return prepareQuery(iterator.toString(), this.getGID(url), getParameter(url,"sort")); + } catch (URISyntaxException e) { + LOGGER.error(String.format("Error changing page: %s", e.getMessage())); + return null; + } + } else { + return null; + } + } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java index 8e3b9d15..a4fc08cc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java @@ -1,223 +1,223 @@ -package com.rarchives.ripme.ripper.rippers; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import org.json.JSONObject; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.Connection.Response; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -/** - * For ripping VSCO pictures. - */ -public class VscoRipper extends AbstractHTMLRipper { - - int pageNumber = 1; - JSONObject profileJSON; - - - private static final String DOMAIN = "vsco.co", - HOST = "vsco"; - - public VscoRipper(URL url) throws IOException{ - super(url); - } - - /** - * Checks to see if VscoRipper can Rip specified url. - * @param url - * @return True if can rip. - * False if cannot rip. - */ - @Override - public boolean canRip(URL url) { - if (!url.getHost().endsWith(DOMAIN)) { - return false; - } - // Ignores personalized things (e.g. login, feed) and store page - // Allows links to user profiles and links to images. - //@TODO: Add support for journals and collections. - String u = url.toExternalForm(); - return !u.contains("/store/") || - !u.contains("/feed/") || - !u.contains("/login/") || - !u.contains("/journal/") || - !u.contains("/collection/")|| - !u.contains("/images/") || - u.contains("/media/"); - - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - //no sanitization needed. - return url; - } - - /** - *

Gets the direct URL of full-sized image through the tag.

- * When expanding future functionality (e.g. support from journals), put everything into this method. - * @param page - * @return - */ - @Override - public List getURLsFromPage(Document page){ - List toRip = new ArrayList<>(); - //If user wanted to rip single image - if (url.toString().contains("/media/")){ - try { - toRip.add(vscoImageToURL(url.toExternalForm())); - } catch (IOException ex) { - LOGGER.debug("Failed to convert " + url.toString() + " to external form."); - } - - } else { - String username = getUserName(); - String userTkn = getUserTkn(username); - String siteID = getSiteID(userTkn, username); - while (true) { - profileJSON = getProfileJSON(userTkn, username, Integer.toString(pageNumber), siteID); - for (int i = 0; i < profileJSON.getJSONArray("media").length(); i++) { - toRip.add("https://" + profileJSON.getJSONArray("media").getJSONObject(i).getString("responsive_url")); - } - if (pageNumber * 1000 > profileJSON.getInt("total")) { - return toRip; - } - pageNumber++; - } - - - } - - return toRip; - } - - private String getUserTkn(String username) { - String userTokenPage = "https://vsco.co/content/Static"; - Map responseCookies = new HashMap<>(); - try { - Response resp = Http.url(userTokenPage).ignoreContentType().response(); - responseCookies = resp.cookies(); - return responseCookies.get("vs"); - } catch (IOException e) { - LOGGER.error("Could not get user tkn"); - return null; - } - } - - private String getUserName() { - Pattern p = Pattern.compile("^https?://vsco.co/([a-zA-Z0-9-]+)(/gallery)?(/)?"); - Matcher m = p.matcher(url.toExternalForm()); - - if (m.matches()) { - String user = m.group(1); - return user; - } - return null; - } - - private JSONObject getProfileJSON(String tkn, String username, String page, String siteId) { - String size = "1000"; - String purl = "https://vsco.co/ajxp/" + tkn + "/2.0/medias?site_id=" + siteId + "&page=" + page + "&size=" + size; - Map cookies = new HashMap<>(); - cookies.put("vs", tkn); - try { - JSONObject j = Http.url(purl).cookies(cookies).getJSON(); - return j; - } catch (IOException e) { - LOGGER.error("Could not profile images"); - return null; - } - } - - private String getSiteID(String tkn, String username) { - Map cookies = new HashMap<>(); - cookies.put("vs", tkn); - try { - JSONObject j = Http.url("https://vsco.co/ajxp/" + tkn + "/2.0/sites?subdomain=" + username).cookies(cookies).getJSON(); - return Integer.toString(j.getJSONArray("sites").getJSONObject(0).getInt("id")); - } catch (IOException e) { - LOGGER.error("Could not get site id"); - return null; - } - } - - private String vscoImageToURL(String url) throws IOException{ - Document page = Jsoup.connect(url).userAgent(USER_AGENT) - .get(); - //create Elements filled only with Elements with the "meta" tag. - Elements metaTags = page.getElementsByTag("meta"); - String result = ""; - - for(Element metaTag : metaTags){ - //find URL inside meta-tag with property of "og:image" - if (metaTag.attr("property").equals("og:image")){ - String givenURL = metaTag.attr("content"); - givenURL = givenURL.replaceAll("\\?h=[0-9]+", "");//replace the "?h=xxx" tag at the end of the URL (where each x is a number) - - result = givenURL; - LOGGER.debug("Found image URL: " + givenURL); - break;//immediately stop after getting URL (there should only be 1 image to be downloaded) - } - } - - //Means website changed, things need to be fixed. - if (result.isEmpty()){ - LOGGER.error("Could not find image URL at: " + url); - } - - return result; - - } - - @Override - public String getHost() { - return HOST; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - - //Single Image - Pattern p = Pattern.compile("^https?://vsco\\.co/([a-zA-Z0-9-]+)/media/([a-zA-Z0-9]+)"); - Matcher m = p.matcher(url.toExternalForm()); - - if (m.matches()){ - // Return the text contained between () in the regex - String user = m.group(1); - String imageNum = m.group(2).substring(0, 5);//first 5 characters should be enough to make each rip unique - return user + "/" + imageNum; - } - - //Member profile (Usernames should all be different, so this should work. - p = Pattern.compile("^https?://vsco.co/([a-zA-Z0-9-]+)(/gallery)?(/)?"); - m = p.matcher(url.toExternalForm()); - - if (m.matches()){ - String user = m.group(1); - return user; - } - - throw new MalformedURLException("Expected a URL to a single image or to a member profile, got " + url + " instead"); - - } - - @Override - public String getDomain() { - return DOMAIN; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } - -} +package com.rarchives.ripme.ripper.rippers; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.json.JSONObject; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.Connection.Response; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +/** + * For ripping VSCO pictures. + */ +public class VscoRipper extends AbstractHTMLRipper { + + int pageNumber = 1; + JSONObject profileJSON; + + + private static final String DOMAIN = "vsco.co", + HOST = "vsco"; + + public VscoRipper(URL url) throws IOException{ + super(url); + } + + /** + * Checks to see if VscoRipper can Rip specified url. + * @param url + * @return True if can rip. + * False if cannot rip. + */ + @Override + public boolean canRip(URL url) { + if (!url.getHost().endsWith(DOMAIN)) { + return false; + } + // Ignores personalized things (e.g. login, feed) and store page + // Allows links to user profiles and links to images. + //@TODO: Add support for journals and collections. + String u = url.toExternalForm(); + return !u.contains("/store/") || + !u.contains("/feed/") || + !u.contains("/login/") || + !u.contains("/journal/") || + !u.contains("/collection/")|| + !u.contains("/images/") || + u.contains("/media/"); + + } + + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + //no sanitization needed. + return url; + } + + /** + *

Gets the direct URL of full-sized image through the tag.

+ * When expanding future functionality (e.g. support from journals), put everything into this method. + * @param page + * @return + */ + @Override + public List getURLsFromPage(Document page){ + List toRip = new ArrayList<>(); + //If user wanted to rip single image + if (url.toString().contains("/media/")){ + try { + toRip.add(vscoImageToURL(url.toExternalForm())); + } catch (IOException ex) { + LOGGER.debug("Failed to convert " + url.toString() + " to external form."); + } + + } else { + String username = getUserName(); + String userTkn = getUserTkn(username); + String siteID = getSiteID(userTkn, username); + while (true) { + profileJSON = getProfileJSON(userTkn, username, Integer.toString(pageNumber), siteID); + for (int i = 0; i < profileJSON.getJSONArray("media").length(); i++) { + toRip.add("https://" + profileJSON.getJSONArray("media").getJSONObject(i).getString("responsive_url")); + } + if (pageNumber * 1000 > profileJSON.getInt("total")) { + return toRip; + } + pageNumber++; + } + + + } + + return toRip; + } + + private String getUserTkn(String username) { + String userTokenPage = "https://vsco.co/content/Static"; + Map responseCookies = new HashMap<>(); + try { + Response resp = Http.url(userTokenPage).ignoreContentType().response(); + responseCookies = resp.cookies(); + return responseCookies.get("vs"); + } catch (IOException e) { + LOGGER.error("Could not get user tkn"); + return null; + } + } + + private String getUserName() { + Pattern p = Pattern.compile("^https?://vsco.co/([a-zA-Z0-9-]+)(/gallery)?(/)?"); + Matcher m = p.matcher(url.toExternalForm()); + + if (m.matches()) { + String user = m.group(1); + return user; + } + return null; + } + + private JSONObject getProfileJSON(String tkn, String username, String page, String siteId) { + String size = "1000"; + String purl = "https://vsco.co/ajxp/" + tkn + "/2.0/medias?site_id=" + siteId + "&page=" + page + "&size=" + size; + Map cookies = new HashMap<>(); + cookies.put("vs", tkn); + try { + JSONObject j = Http.url(purl).cookies(cookies).getJSON(); + return j; + } catch (IOException e) { + LOGGER.error("Could not profile images"); + return null; + } + } + + private String getSiteID(String tkn, String username) { + Map cookies = new HashMap<>(); + cookies.put("vs", tkn); + try { + JSONObject j = Http.url("https://vsco.co/ajxp/" + tkn + "/2.0/sites?subdomain=" + username).cookies(cookies).getJSON(); + return Integer.toString(j.getJSONArray("sites").getJSONObject(0).getInt("id")); + } catch (IOException e) { + LOGGER.error("Could not get site id"); + return null; + } + } + + private String vscoImageToURL(String url) throws IOException{ + Document page = Jsoup.connect(url).userAgent(USER_AGENT) + .get(); + //create Elements filled only with Elements with the "meta" tag. + Elements metaTags = page.getElementsByTag("meta"); + String result = ""; + + for(Element metaTag : metaTags){ + //find URL inside meta-tag with property of "og:image" + if (metaTag.attr("property").equals("og:image")){ + String givenURL = metaTag.attr("content"); + givenURL = givenURL.replaceAll("\\?h=[0-9]+", "");//replace the "?h=xxx" tag at the end of the URL (where each x is a number) + + result = givenURL; + LOGGER.debug("Found image URL: " + givenURL); + break;//immediately stop after getting URL (there should only be 1 image to be downloaded) + } + } + + //Means website changed, things need to be fixed. + if (result.isEmpty()){ + LOGGER.error("Could not find image URL at: " + url); + } + + return result; + + } + + @Override + public String getHost() { + return HOST; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + + //Single Image + Pattern p = Pattern.compile("^https?://vsco\\.co/([a-zA-Z0-9-]+)/media/([a-zA-Z0-9]+)"); + Matcher m = p.matcher(url.toExternalForm()); + + if (m.matches()){ + // Return the text contained between () in the regex + String user = m.group(1); + String imageNum = m.group(2).substring(0, 5);//first 5 characters should be enough to make each rip unique + return user + "/" + imageNum; + } + + //Member profile (Usernames should all be different, so this should work. + p = Pattern.compile("^https?://vsco.co/([a-zA-Z0-9-]+)(/gallery)?(/)?"); + m = p.matcher(url.toExternalForm()); + + if (m.matches()){ + String user = m.group(1); + return user; + } + + throw new MalformedURLException("Expected a URL to a single image or to a member profile, got " + url + " instead"); + + } + + @Override + public String getDomain() { + return DOMAIN; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XlecxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XlecxRipper.java index 0aaacfc4..2e95c04a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XlecxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XlecxRipper.java @@ -1,36 +1,36 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class XlecxRipper extends XcartxRipper { - - private Pattern p = Pattern.compile("^https?://xlecx.org/([a-zA-Z0-9_\\-]+).html"); - - public XlecxRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "xlecx"; - } - - @Override - public String getDomain() { - return "xlecx.org"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - throw new MalformedURLException("Expected URL format: http://xlecx.org/comic, got: " + url); - - } -} +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class XlecxRipper extends XcartxRipper { + + private Pattern p = Pattern.compile("^https?://xlecx.org/([a-zA-Z0-9_\\-]+).html"); + + public XlecxRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "xlecx"; + } + + @Override + public String getDomain() { + return "xlecx.org"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected URL format: http://xlecx.org/comic, got: " + url); + + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java index d977708a..c72a5a59 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java @@ -1,80 +1,80 @@ -package com.rarchives.ripme.ripper.rippers.video; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - - -import com.rarchives.ripme.ripper.VideoRipper; -import com.rarchives.ripme.utils.Http; - -public class TwitchVideoRipper extends VideoRipper { - - private static final String HOST = "twitch"; - - public TwitchVideoRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return HOST; - } - - @Override - public boolean canRip(URL url) { - Pattern p = Pattern.compile("^https://clips\\.twitch\\.tv/.*$"); - Matcher m = p.matcher(url.toExternalForm()); - return m.matches(); - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return url; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https://clips\\.twitch\\.tv/(.*)$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(m.groupCount()); - } - - throw new MalformedURLException( - "Expected Twitch.tv format:" - + "https://clips.twitch.tv/####" - + " Got: " + url); - } - - @Override - public void rip() throws IOException { - LOGGER.info("Retrieving " + this.url); - Document doc = Http.url(url).get(); - - //Get user friendly filename from page title - String title = doc.title(); - - Elements script = doc.select("script"); - if (script.isEmpty()) { - throw new IOException("Could not find script code at " + url); - } - //Regex assumes highest quality source is listed first - Pattern p = Pattern.compile("\"source\":\"(.*?)\""); - - for (Element element : script) { - Matcher m = p.matcher(element.data()); - if (m.find()){ - String vidUrl = m.group(1); - addURLToDownload(new URL(vidUrl), HOST + "_" + title); - } - } - waitForThreads(); - } +package com.rarchives.ripme.ripper.rippers.video; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + + +import com.rarchives.ripme.ripper.VideoRipper; +import com.rarchives.ripme.utils.Http; + +public class TwitchVideoRipper extends VideoRipper { + + private static final String HOST = "twitch"; + + public TwitchVideoRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return HOST; + } + + @Override + public boolean canRip(URL url) { + Pattern p = Pattern.compile("^https://clips\\.twitch\\.tv/.*$"); + Matcher m = p.matcher(url.toExternalForm()); + return m.matches(); + } + + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + return url; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^https://clips\\.twitch\\.tv/(.*)$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(m.groupCount()); + } + + throw new MalformedURLException( + "Expected Twitch.tv format:" + + "https://clips.twitch.tv/####" + + " Got: " + url); + } + + @Override + public void rip() throws IOException { + LOGGER.info("Retrieving " + this.url); + Document doc = Http.url(url).get(); + + //Get user friendly filename from page title + String title = doc.title(); + + Elements script = doc.select("script"); + if (script.isEmpty()) { + throw new IOException("Could not find script code at " + url); + } + //Regex assumes highest quality source is listed first + Pattern p = Pattern.compile("\"source\":\"(.*?)\""); + + for (Element element : script) { + Matcher m = p.matcher(element.data()); + if (m.find()){ + String vidUrl = m.group(1); + addURLToDownload(new URL(vidUrl), HOST + "_" + title); + } + } + waitForThreads(); + } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/utils/Proxy.java b/src/main/java/com/rarchives/ripme/utils/Proxy.java index be3c3b7e..0275bd5c 100644 --- a/src/main/java/com/rarchives/ripme/utils/Proxy.java +++ b/src/main/java/com/rarchives/ripme/utils/Proxy.java @@ -1,99 +1,99 @@ -package com.rarchives.ripme.utils; - -import java.net.Authenticator; -import java.net.PasswordAuthentication; -import java.util.Map; -import java.util.HashMap; - -/** - * Proxy/Socks setter - */ -public class Proxy { - private Proxy() { - } - - /** - * Parse the proxy server settings from string, using the format - * [user:password]@host[:port]. - * - * @param fullproxy the string to parse - * @return HashMap containing proxy server, port, user and password - */ - private static Map parseServer(String fullproxy) { - Map proxy = new HashMap(); - - if (fullproxy.lastIndexOf("@") != -1) { - int sservli = fullproxy.lastIndexOf("@"); - String userpw = fullproxy.substring(0, sservli); - String[] usersplit = userpw.split(":"); - proxy.put("user", usersplit[0]); - proxy.put("password", usersplit[1]); - fullproxy = fullproxy.substring(sservli + 1); - } - String[] servsplit = fullproxy.split(":"); - if (servsplit.length == 2) { - proxy.put("port", servsplit[1]); - } - proxy.put("server", servsplit[0]); - return proxy; - } - - /** - * Set a HTTP Proxy. - * WARNING: Authenticated HTTP Proxy won't work from jdk1.8.111 unless - * passing the flag -Djdk.http.auth.tunneling.disabledSchemes="" to java - * see https://stackoverflow.com/q/41505219 - * - * @param fullproxy the proxy, using format [user:password]@host[:port] - */ - public static void setHTTPProxy(String fullproxy) { - Map proxyServer = parseServer(fullproxy); - - if (proxyServer.get("user") != null && proxyServer.get("password") != null) { - Authenticator.setDefault(new Authenticator(){ - protected PasswordAuthentication getPasswordAuthentication(){ - PasswordAuthentication p = new PasswordAuthentication(proxyServer.get("user"), proxyServer.get("password").toCharArray()); - return p; - } - }); - System.setProperty("http.proxyUser", proxyServer.get("user")); - System.setProperty("http.proxyPassword", proxyServer.get("password")); - System.setProperty("https.proxyUser", proxyServer.get("user")); - System.setProperty("https.proxyPassword", proxyServer.get("password")); - } - - if (proxyServer.get("port") != null) { - System.setProperty("http.proxyPort", proxyServer.get("port")); - System.setProperty("https.proxyPort", proxyServer.get("port")); - } - - System.setProperty("http.proxyHost", proxyServer.get("server")); - System.setProperty("https.proxyHost", proxyServer.get("server")); - } - - /** - * Set a Socks Proxy Server (globally). - * - * @param fullsocks the socks server, using format [user:password]@host[:port] - */ - public static void setSocks(String fullsocks) { - - Map socksServer = parseServer(fullsocks); - if (socksServer.get("user") != null && socksServer.get("password") != null) { - Authenticator.setDefault(new Authenticator(){ - protected PasswordAuthentication getPasswordAuthentication(){ - PasswordAuthentication p = new PasswordAuthentication(socksServer.get("user"), socksServer.get("password").toCharArray()); - return p; - } - }); - System.setProperty("java.net.socks.username", socksServer.get("user")); - System.setProperty("java.net.socks.password", socksServer.get("password")); - } - if (socksServer.get("port") != null) { - System.setProperty("socksProxyPort", socksServer.get("port")); - } - - System.setProperty("socksProxyHost", socksServer.get("server")); - } - -} +package com.rarchives.ripme.utils; + +import java.net.Authenticator; +import java.net.PasswordAuthentication; +import java.util.Map; +import java.util.HashMap; + +/** + * Proxy/Socks setter + */ +public class Proxy { + private Proxy() { + } + + /** + * Parse the proxy server settings from string, using the format + * [user:password]@host[:port]. + * + * @param fullproxy the string to parse + * @return HashMap containing proxy server, port, user and password + */ + private static Map parseServer(String fullproxy) { + Map proxy = new HashMap(); + + if (fullproxy.lastIndexOf("@") != -1) { + int sservli = fullproxy.lastIndexOf("@"); + String userpw = fullproxy.substring(0, sservli); + String[] usersplit = userpw.split(":"); + proxy.put("user", usersplit[0]); + proxy.put("password", usersplit[1]); + fullproxy = fullproxy.substring(sservli + 1); + } + String[] servsplit = fullproxy.split(":"); + if (servsplit.length == 2) { + proxy.put("port", servsplit[1]); + } + proxy.put("server", servsplit[0]); + return proxy; + } + + /** + * Set a HTTP Proxy. + * WARNING: Authenticated HTTP Proxy won't work from jdk1.8.111 unless + * passing the flag -Djdk.http.auth.tunneling.disabledSchemes="" to java + * see https://stackoverflow.com/q/41505219 + * + * @param fullproxy the proxy, using format [user:password]@host[:port] + */ + public static void setHTTPProxy(String fullproxy) { + Map proxyServer = parseServer(fullproxy); + + if (proxyServer.get("user") != null && proxyServer.get("password") != null) { + Authenticator.setDefault(new Authenticator(){ + protected PasswordAuthentication getPasswordAuthentication(){ + PasswordAuthentication p = new PasswordAuthentication(proxyServer.get("user"), proxyServer.get("password").toCharArray()); + return p; + } + }); + System.setProperty("http.proxyUser", proxyServer.get("user")); + System.setProperty("http.proxyPassword", proxyServer.get("password")); + System.setProperty("https.proxyUser", proxyServer.get("user")); + System.setProperty("https.proxyPassword", proxyServer.get("password")); + } + + if (proxyServer.get("port") != null) { + System.setProperty("http.proxyPort", proxyServer.get("port")); + System.setProperty("https.proxyPort", proxyServer.get("port")); + } + + System.setProperty("http.proxyHost", proxyServer.get("server")); + System.setProperty("https.proxyHost", proxyServer.get("server")); + } + + /** + * Set a Socks Proxy Server (globally). + * + * @param fullsocks the socks server, using format [user:password]@host[:port] + */ + public static void setSocks(String fullsocks) { + + Map socksServer = parseServer(fullsocks); + if (socksServer.get("user") != null && socksServer.get("password") != null) { + Authenticator.setDefault(new Authenticator(){ + protected PasswordAuthentication getPasswordAuthentication(){ + PasswordAuthentication p = new PasswordAuthentication(socksServer.get("user"), socksServer.get("password").toCharArray()); + return p; + } + }); + System.setProperty("java.net.socks.username", socksServer.get("user")); + System.setProperty("java.net.socks.password", socksServer.get("password")); + } + if (socksServer.get("port") != null) { + System.setProperty("socksProxyPort", socksServer.get("port")); + } + + System.setProperty("socksProxyHost", socksServer.get("server")); + } + +} diff --git a/src/main/resources/LabelsBundle_pl_PL.properties b/src/main/resources/LabelsBundle_pl_PL.properties index 4ba4590e..dbb74ef1 100644 --- a/src/main/resources/LabelsBundle_pl_PL.properties +++ b/src/main/resources/LabelsBundle_pl_PL.properties @@ -1,59 +1,59 @@ -Log = Logi -History = Historia -created = Stworzono -modified = Zmodyfikowano -queue = Kolejka -Configuration = Konfiguracja - -# Keys for the Configuration menu - -current.version = Obecna Wersja -check.for.updates = Sprawdź dostępność aktualizacji -auto.update = Auto Aktualizacja? -max.download.threads = Maksymalna Ilośc Pobieranych Plików: -timeout.mill = Opóźnienie (w milisekundach): -retry.download.count = Liczba ponownych pobrań -overwrite.existing.files = Nadpisać istniejące pliki? -sound.when.rip.completes = Dźwięk po zakończeniu -preserve.order = Zachować porządek -save.logs = Zapisz Logi -notification.when.rip.starts = Powiadomienie przy uruchomieniu pobierania -save.urls.only = Zapisz tylko linki -save.album.titles = Zapisz nazwy albumów -autorip.from.clipboard = Auto pobieranie ze schowka -save.descriptions = Zapisz opis -prefer.mp4.over.gif = Preferuj MP4 od GIF -restore.window.position = Przywróć pozycję okna -remember.url.history = Zapamiętaj historię linków -loading.history.from = Załaduj historię z... - -# Misc UI keys - -loading.history.from.configuration = Załaduj historię z ustawień -interrupted.while.waiting.to.rip.next.album = Przerwany podczas oczekiwania na zgrywanie następnego albumu -inactive = Nieaktywny -re-rip.checked = Sprawdź pobrane ripy -remove = Usuń -clear = Wyczyść -download.url.list = Pobierz listę linków -select.save.dir = Wybierz ścieżkę zapisu - -# Keys for the logs generated by DownloadFileThread - -nonretriable.status.code = Nieodwracalny kod statusu -retriable.status.code = Odzyskiwanie kodu statusu -server.doesnt.support.resuming.downloads = Serwer nie obsługuje wznowienia pobierania - -# A "magic number" can also be called a file signature - -was.unable.to.get.content.type.using.magic.number = Nie udało się uzyskać typu zawartości za pomocą magicznej liczby -magic.number.was = Magiczną liczbą była -deleting.existing.file = Usuwanie istniejących plików -request.properties = Poproś o uprawnienia -download.interrupted = Pobieranie przerwane -exceeded.maximum.retries = Spodziewana ilośc powtórzeń -http.status.exception = Wyjątek statusu http -exception.while.downloading.file = Wystąpił problem podczas pobierania pliku -failed.to.download = Nie można pobrać pliku -skipping = Pomijanie +Log = Logi +History = Historia +created = Stworzono +modified = Zmodyfikowano +queue = Kolejka +Configuration = Konfiguracja + +# Keys for the Configuration menu + +current.version = Obecna Wersja +check.for.updates = Sprawdź dostępność aktualizacji +auto.update = Auto Aktualizacja? +max.download.threads = Maksymalna Ilośc Pobieranych Plików: +timeout.mill = Opóźnienie (w milisekundach): +retry.download.count = Liczba ponownych pobrań +overwrite.existing.files = Nadpisać istniejące pliki? +sound.when.rip.completes = Dźwięk po zakończeniu +preserve.order = Zachować porządek +save.logs = Zapisz Logi +notification.when.rip.starts = Powiadomienie przy uruchomieniu pobierania +save.urls.only = Zapisz tylko linki +save.album.titles = Zapisz nazwy albumów +autorip.from.clipboard = Auto pobieranie ze schowka +save.descriptions = Zapisz opis +prefer.mp4.over.gif = Preferuj MP4 od GIF +restore.window.position = Przywróć pozycję okna +remember.url.history = Zapamiętaj historię linków +loading.history.from = Załaduj historię z... + +# Misc UI keys + +loading.history.from.configuration = Załaduj historię z ustawień +interrupted.while.waiting.to.rip.next.album = Przerwany podczas oczekiwania na zgrywanie następnego albumu +inactive = Nieaktywny +re-rip.checked = Sprawdź pobrane ripy +remove = Usuń +clear = Wyczyść +download.url.list = Pobierz listę linków +select.save.dir = Wybierz ścieżkę zapisu + +# Keys for the logs generated by DownloadFileThread + +nonretriable.status.code = Nieodwracalny kod statusu +retriable.status.code = Odzyskiwanie kodu statusu +server.doesnt.support.resuming.downloads = Serwer nie obsługuje wznowienia pobierania + +# A "magic number" can also be called a file signature + +was.unable.to.get.content.type.using.magic.number = Nie udało się uzyskać typu zawartości za pomocą magicznej liczby +magic.number.was = Magiczną liczbą była +deleting.existing.file = Usuwanie istniejących plików +request.properties = Poproś o uprawnienia +download.interrupted = Pobieranie przerwane +exceeded.maximum.retries = Spodziewana ilośc powtórzeń +http.status.exception = Wyjątek statusu http +exception.while.downloading.file = Wystąpił problem podczas pobierania pliku +failed.to.download = Nie można pobrać pliku +skipping = Pomijanie file.already.exists = Plik już istnieje \ No newline at end of file diff --git a/src/main/resources/LabelsBundle_zh_CN.properties b/src/main/resources/LabelsBundle_zh_CN.properties index 994efcf8..cd43da1c 100644 --- a/src/main/resources/LabelsBundle_zh_CN.properties +++ b/src/main/resources/LabelsBundle_zh_CN.properties @@ -1,75 +1,75 @@ -Log = 日志 -History = 历史 -created = 创建时间 -modified = 修改时间 -queue = 队列 -Configuration = 配置 -open = 打开 - -# Keys for the Configuration menu -current.version = 当前版本 -check.for.updates = 检查更新 -auto.update = 自动更新？ -max.download.threads = 最大下载线程数： -timeout.mill = 超时（毫秒）： -retry.download.count = 重试下载次数 -overwrite.existing.files = 覆盖现有文件？ -sound.when.rip.completes = 抓取完成时播放声音 -preserve.order = 保持顺序 -save.logs = 保存日志 -notification.when.rip.starts = 通知抓取开始 -save.urls.only = 仅保存 URL -save.album.titles = 保存专辑标题 -autorip.from.clipboard = 监视剪贴板上的 URL -save.descriptions = 保存描述 -prefer.mp4.over.gif = 首选 MP4 而非 GIF -restore.window.position = 恢复窗口位置 -remember.url.history = 记住 URL 历史 -loading.history.from = 加载历史从 - -# Queue keys -queue.remove.all = 移除全部 -queue.validation = 您确定要移除队列内的全部项目？ -queue.remove.selected = 移除所选项目 - -# History -re-rip.checked = 重新抓取选中的项目 -remove = 移除 -clear = 清除 -history.check.all = 选中全部 -history.check.none = 取消选中全部 -history.check.selected = 选中所选项目 -history.uncheck.selected = 取消选中所选项目 -history.load.failed.warning = RipMe 加载位于 historyFile.getAbsolutePath() 的历史文件失败\n\n错误：%s\n\n关闭 RipMe 会自动覆盖此文件的内容，\n请在关闭 RipMe 前备份它！ -history.load.none = 无可重新抓取的历史条目。请先抓取一些专辑 -history.load.none.checked = 未 '选中' 任何历史条目，请通过选中所需 URL 前面的复选框或URL 的右键菜单以选中所需条目 - -# TrayIcon -tray.show = 显示 -tray.hide = 隐藏 -tray.autorip = 监视剪贴板上的 URL -tray.exit = 退出 - -# Misc UI keys -loading.history.from.configuration = 从配置加载历史 -interrupted.while.waiting.to.rip.next.album = 等候抓取下一专辑期间发生中断 -inactive = 非活动 -download.url.list = 下载 URL 列表 -select.save.dir = 选择保存目录 - -# Keys for the logs generated by DownloadFileThread -nonretriable.status.code = 非可重试状态代码 -retriable.status.code = 可重试状态代码 -server.doesnt.support.resuming.downloads = 服务器不支持继续下载（续传） -# A "magic number" can also be called a file signature -was.unable.to.get.content.type.using.magic.number = 不能使用幻数获取内容类型 -magic.number.was = 幻数为 -deleting.existing.file = 删除现有文件 -request.properties = 请求属性 -download.interrupted = 下载中断 -exceeded.maximum.retries = 超过最大重试次数 -http.status.exception = HTTP 状态意外 -exception.while.downloading.file = 下载文件时发生意外 -failed.to.download = 下载失败 -skipping = 跳过 +Log = 日志 +History = 历史 +created = 创建时间 +modified = 修改时间 +queue = 队列 +Configuration = 配置 +open = 打开 + +# Keys for the Configuration menu +current.version = 当前版本 +check.for.updates = 检查更新 +auto.update = 自动更新？ +max.download.threads = 最大下载线程数： +timeout.mill = 超时（毫秒）： +retry.download.count = 重试下载次数 +overwrite.existing.files = 覆盖现有文件？ +sound.when.rip.completes = 抓取完成时播放声音 +preserve.order = 保持顺序 +save.logs = 保存日志 +notification.when.rip.starts = 通知抓取开始 +save.urls.only = 仅保存 URL +save.album.titles = 保存专辑标题 +autorip.from.clipboard = 监视剪贴板上的 URL +save.descriptions = 保存描述 +prefer.mp4.over.gif = 首选 MP4 而非 GIF +restore.window.position = 恢复窗口位置 +remember.url.history = 记住 URL 历史 +loading.history.from = 加载历史从 + +# Queue keys +queue.remove.all = 移除全部 +queue.validation = 您确定要移除队列内的全部项目？ +queue.remove.selected = 移除所选项目 + +# History +re-rip.checked = 重新抓取选中的项目 +remove = 移除 +clear = 清除 +history.check.all = 选中全部 +history.check.none = 取消选中全部 +history.check.selected = 选中所选项目 +history.uncheck.selected = 取消选中所选项目 +history.load.failed.warning = RipMe 加载位于 historyFile.getAbsolutePath() 的历史文件失败\n\n错误：%s\n\n关闭 RipMe 会自动覆盖此文件的内容，\n请在关闭 RipMe 前备份它！ +history.load.none = 无可重新抓取的历史条目。请先抓取一些专辑 +history.load.none.checked = 未 '选中' 任何历史条目，请通过选中所需 URL 前面的复选框或URL 的右键菜单以选中所需条目 + +# TrayIcon +tray.show = 显示 +tray.hide = 隐藏 +tray.autorip = 监视剪贴板上的 URL +tray.exit = 退出 + +# Misc UI keys +loading.history.from.configuration = 从配置加载历史 +interrupted.while.waiting.to.rip.next.album = 等候抓取下一专辑期间发生中断 +inactive = 非活动 +download.url.list = 下载 URL 列表 +select.save.dir = 选择保存目录 + +# Keys for the logs generated by DownloadFileThread +nonretriable.status.code = 非可重试状态代码 +retriable.status.code = 可重试状态代码 +server.doesnt.support.resuming.downloads = 服务器不支持继续下载（续传） +# A "magic number" can also be called a file signature +was.unable.to.get.content.type.using.magic.number = 不能使用幻数获取内容类型 +magic.number.was = 幻数为 +deleting.existing.file = 删除现有文件 +request.properties = 请求属性 +download.interrupted = 下载中断 +exceeded.maximum.retries = 超过最大重试次数 +http.status.exception = HTTP 状态意外 +exception.while.downloading.file = 下载文件时发生意外 +failed.to.download = 下载失败 +skipping = 跳过 file.already.exists = 文件已存在 \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/proxyTest.java b/src/test/java/com/rarchives/ripme/tst/proxyTest.java index 7791290a..0576b8e2 100644 --- a/src/test/java/com/rarchives/ripme/tst/proxyTest.java +++ b/src/test/java/com/rarchives/ripme/tst/proxyTest.java @@ -1,57 +1,57 @@ -package com.rarchives.ripme.tst; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import com.rarchives.ripme.utils.Proxy; -import com.rarchives.ripme.utils.Utils; -import com.rarchives.ripme.utils.Http; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertFalse; - -public class proxyTest { - - - // This test will only run on machines where the user has added a entry for proxy.socks - @Test - public void testSocksProxy() throws IOException, URISyntaxException { - // Unset proxy before testing - System.setProperty("http.proxyHost", ""); - System.setProperty("https.proxyHost", ""); - System.setProperty("socksProxyHost", ""); - URL url = new URI("https://icanhazip.com").toURL(); - String proxyConfig = Utils.getConfigString("proxy.socks", ""); - if (!proxyConfig.equals("")) { - String ip1 = Http.url(url).ignoreContentType().get().text(); - Proxy.setSocks(Utils.getConfigString("proxy.socks", "")); - String ip2 = Http.url(url).ignoreContentType().get().text(); - assertFalse(ip1.equals(ip2)); - } else { - System.out.println("Skipping testSocksProxy"); - assert(true); - } - } - - // This test will only run on machines where the user has added a entry for proxy.http - @Test - public void testHTTPProxy() throws IOException, URISyntaxException { - // Unset proxy before testing - System.setProperty("http.proxyHost", ""); - System.setProperty("https.proxyHost", ""); - System.setProperty("socksProxyHost", ""); - URL url = new URI("https://icanhazip.com").toURL(); - String proxyConfig = Utils.getConfigString("proxy.http", ""); - if (!proxyConfig.equals("")) { - String ip1 = Http.url(url).ignoreContentType().get().text(); - Proxy.setHTTPProxy(Utils.getConfigString("proxy.http", "")); - String ip2 = Http.url(url).ignoreContentType().get().text(); - assertFalse(ip1.equals(ip2)); - } else { - System.out.println("Skipping testHTTPProxy"); - assert(true); - } - } - -} +package com.rarchives.ripme.tst; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import com.rarchives.ripme.utils.Proxy; +import com.rarchives.ripme.utils.Utils; +import com.rarchives.ripme.utils.Http; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; + +public class proxyTest { + + + // This test will only run on machines where the user has added a entry for proxy.socks + @Test + public void testSocksProxy() throws IOException, URISyntaxException { + // Unset proxy before testing + System.setProperty("http.proxyHost", ""); + System.setProperty("https.proxyHost", ""); + System.setProperty("socksProxyHost", ""); + URL url = new URI("https://icanhazip.com").toURL(); + String proxyConfig = Utils.getConfigString("proxy.socks", ""); + if (!proxyConfig.equals("")) { + String ip1 = Http.url(url).ignoreContentType().get().text(); + Proxy.setSocks(Utils.getConfigString("proxy.socks", "")); + String ip2 = Http.url(url).ignoreContentType().get().text(); + assertFalse(ip1.equals(ip2)); + } else { + System.out.println("Skipping testSocksProxy"); + assert(true); + } + } + + // This test will only run on machines where the user has added a entry for proxy.http + @Test + public void testHTTPProxy() throws IOException, URISyntaxException { + // Unset proxy before testing + System.setProperty("http.proxyHost", ""); + System.setProperty("https.proxyHost", ""); + System.setProperty("socksProxyHost", ""); + URL url = new URI("https://icanhazip.com").toURL(); + String proxyConfig = Utils.getConfigString("proxy.http", ""); + if (!proxyConfig.equals("")) { + String ip1 = Http.url(url).ignoreContentType().get().text(); + Proxy.setHTTPProxy(Utils.getConfigString("proxy.http", "")); + String ip2 = Http.url(url).ignoreContentType().get().text(); + assertFalse(ip1.equals(ip2)); + } else { + System.out.println("Skipping testHTTPProxy"); + assert(true); + } + } + +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java index 0a64487d..e01ae6e0 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java @@ -1,28 +1,28 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import com.rarchives.ripme.ripper.rippers.ComicextraRipper; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -public class ComicextraRipperTest extends RippersTest { - @Test - @Tag("flaky") - public void testComicUrl() throws IOException, URISyntaxException { - URL url = new URI("https://www.comicextra.com/comic/karma-police").toURL(); - ComicextraRipper ripper = new ComicextraRipper(url); - testRipper(ripper); - } - @Test - @Disabled("no images found error, broken ripper?") - public void testChapterUrl() throws IOException, URISyntaxException { - URL url = new URI("https://www.comicextra.com/v-for-vendetta/chapter-1").toURL(); - ComicextraRipper ripper = new ComicextraRipper(url); - testRipper(ripper); - } - -} +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import com.rarchives.ripme.ripper.rippers.ComicextraRipper; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +public class ComicextraRipperTest extends RippersTest { + @Test + @Tag("flaky") + public void testComicUrl() throws IOException, URISyntaxException { + URL url = new URI("https://www.comicextra.com/comic/karma-police").toURL(); + ComicextraRipper ripper = new ComicextraRipper(url); + testRipper(ripper); + } + @Test + @Disabled("no images found error, broken ripper?") + public void testChapterUrl() throws IOException, URISyntaxException { + URL url = new URI("https://www.comicextra.com/v-for-vendetta/chapter-1").toURL(); + ComicextraRipper ripper = new ComicextraRipper(url); + testRipper(ripper); + } + +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java index 17f0c8a6..14fcef07 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java @@ -1,55 +1,55 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import com.rarchives.ripme.ripper.rippers.CyberdropRipper; -import com.rarchives.ripme.utils.Http; -import org.jsoup.nodes.Document; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class CyberdropRipperTest extends RippersTest { - @Test - public void testScrolllerGID() throws IOException, URISyntaxException { - Map testURLs = new HashMap<>(); - - testURLs.put(new URI("https://cyberdrop.me/a/n4umdBjw").toURL(), "n4umdBjw"); - testURLs.put(new URI("https://cyberdrop.me/a/iLtp4BjW").toURL(), "iLtp4BjW"); - for (URL url : testURLs.keySet()) { - CyberdropRipper ripper = new CyberdropRipper(url); - ripper.setup(); - Assertions.assertEquals(testURLs.get(url), ripper.getGID(ripper.getURL())); - deleteDir(ripper.getWorkingDir()); - } - } - - @Test - @Tag("flaky") - public void testCyberdropNumberOfFiles() throws IOException, URISyntaxException { - List testURLs = new ArrayList(); - - testURLs.add(new URI("https://cyberdrop.me/a/n4umdBjw").toURL()); - testURLs.add(new URI("https://cyberdrop.me/a/iLtp4BjW").toURL()); - for (URL url : testURLs) { - Assertions.assertTrue(willDownloadAllFiles(url)); - } - } - - public boolean willDownloadAllFiles(URL url) throws IOException { - Document doc = Http.url(url).get(); - long numberOfLinks = doc.getElementsByClass("image").stream().count(); - int numberOfFiles = Integer.parseInt(doc.getElementById("totalFilesAmount").text()); - return numberOfLinks == numberOfFiles; - } - - - +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.CyberdropRipper; +import com.rarchives.ripme.utils.Http; +import org.jsoup.nodes.Document; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class CyberdropRipperTest extends RippersTest { + @Test + public void testScrolllerGID() throws IOException, URISyntaxException { + Map testURLs = new HashMap<>(); + + testURLs.put(new URI("https://cyberdrop.me/a/n4umdBjw").toURL(), "n4umdBjw"); + testURLs.put(new URI("https://cyberdrop.me/a/iLtp4BjW").toURL(), "iLtp4BjW"); + for (URL url : testURLs.keySet()) { + CyberdropRipper ripper = new CyberdropRipper(url); + ripper.setup(); + Assertions.assertEquals(testURLs.get(url), ripper.getGID(ripper.getURL())); + deleteDir(ripper.getWorkingDir()); + } + } + + @Test + @Tag("flaky") + public void testCyberdropNumberOfFiles() throws IOException, URISyntaxException { + List testURLs = new ArrayList(); + + testURLs.add(new URI("https://cyberdrop.me/a/n4umdBjw").toURL()); + testURLs.add(new URI("https://cyberdrop.me/a/iLtp4BjW").toURL()); + for (URL url : testURLs) { + Assertions.assertTrue(willDownloadAllFiles(url)); + } + } + + public boolean willDownloadAllFiles(URL url) throws IOException { + Document doc = Http.url(url).get(); + long numberOfLinks = doc.getElementsByClass("image").stream().count(); + int numberOfFiles = Integer.parseInt(doc.getElementById("totalFilesAmount").text()); + return numberOfLinks == numberOfFiles; + } + + + } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java index cc418842..d4a51a68 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java @@ -1,30 +1,30 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import com.rarchives.ripme.ripper.rippers.FolioRipper; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -public class FolioRipperTest extends RippersTest { - /** - * Test for folio.ink ripper - */ - @Test - @Disabled("test or ripper broken") - public void testFolioRip() throws IOException, URISyntaxException { - FolioRipper ripper = new FolioRipper(new URI("https://folio.ink/DmBe6i").toURL()); - testRipper(ripper); - } - - @Test - public void testGetGID() throws IOException, URISyntaxException { - URL url = new URI("https://folio.ink/DmBe6i").toURL(); - FolioRipper ripper = new FolioRipper(url); - Assertions.assertEquals("DmBe6i", ripper.getGID(url)); - } -} +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import com.rarchives.ripme.ripper.rippers.FolioRipper; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +public class FolioRipperTest extends RippersTest { + /** + * Test for folio.ink ripper + */ + @Test + @Disabled("test or ripper broken") + public void testFolioRip() throws IOException, URISyntaxException { + FolioRipper ripper = new FolioRipper(new URI("https://folio.ink/DmBe6i").toURL()); + testRipper(ripper); + } + + @Test + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://folio.ink/DmBe6i").toURL(); + FolioRipper ripper = new FolioRipper(url); + Assertions.assertEquals("DmBe6i", ripper.getGID(url)); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java index ac06ceb6..c8400d56 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java @@ -1,53 +1,53 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import com.rarchives.ripme.ripper.rippers.GfycatRipper; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - - -public class GfycatRipperTest extends RippersTest { - - /** - * Rips correctly formatted URL directly from Gfycat - */ - @Test - public void testGfycatGoodURL() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/TemptingExcellentIchthyosaurs").toURL()); - testRipper(ripper); - } - /** - * Rips badly formatted URL directly from Gfycat - */ - public void testGfycatBadURL() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/gifs/detail/limitedtestyamericancrow").toURL()); - testRipper(ripper); - } - - /** - * Rips a Gfycat profile - */ - public void testGfycatProfile() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/@golbanstorage").toURL()); - testRipper(ripper); - } - - /** - * Rips a Gfycat amp link - * @throws IOException - */ - public void testGfycatAmp() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/amp/TemptingExcellentIchthyosaurs").toURL()); - testRipper(ripper); - } - - /** - * Rips a Gfycat profile with special characters in username - */ - public void testGfycatSpecialChar() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/@rsss.kr").toURL()); - testRipper(ripper); - } -} +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.GfycatRipper; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + + +public class GfycatRipperTest extends RippersTest { + + /** + * Rips correctly formatted URL directly from Gfycat + */ + @Test + public void testGfycatGoodURL() throws IOException, URISyntaxException { + GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/TemptingExcellentIchthyosaurs").toURL()); + testRipper(ripper); + } + /** + * Rips badly formatted URL directly from Gfycat + */ + public void testGfycatBadURL() throws IOException, URISyntaxException { + GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/gifs/detail/limitedtestyamericancrow").toURL()); + testRipper(ripper); + } + + /** + * Rips a Gfycat profile + */ + public void testGfycatProfile() throws IOException, URISyntaxException { + GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/@golbanstorage").toURL()); + testRipper(ripper); + } + + /** + * Rips a Gfycat amp link + * @throws IOException + */ + public void testGfycatAmp() throws IOException, URISyntaxException { + GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/amp/TemptingExcellentIchthyosaurs").toURL()); + testRipper(ripper); + } + + /** + * Rips a Gfycat profile with special characters in username + */ + public void testGfycatSpecialChar() throws IOException, URISyntaxException { + GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/@rsss.kr").toURL()); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java index ca3aee41..4516f2c8 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java @@ -1,42 +1,42 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - -import com.rarchives.ripme.ripper.rippers.ListalRipper; -import org.junit.jupiter.api.Test; - -public class ListalRipperTest extends RippersTest { - - /** - * Test for list type url. - */ - @Test - public void testPictures() throws IOException, URISyntaxException { - ListalRipper ripper = - new ListalRipper(new URI("https://www.listal.com/emma-stone_iii/pictures").toURL()); - testRipper(ripper); - } - - /** - * Test for list type url. - */ - @Test - public void testRipListType() throws IOException, URISyntaxException { - ListalRipper ripper = - new ListalRipper(new URI("https://www.listal.com/list/evolution-emma-stone").toURL()); - testRipper(ripper); - } - - /** - * Test for folder type url. - */ - @Test - public void testRipFolderType() throws IOException, URISyntaxException { - ListalRipper ripper = - new ListalRipper(new URI("https://www.listal.com/chet-atkins/pictures").toURL()); - testRipper(ripper); - } - -} +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +import com.rarchives.ripme.ripper.rippers.ListalRipper; +import org.junit.jupiter.api.Test; + +public class ListalRipperTest extends RippersTest { + + /** + * Test for list type url. + */ + @Test + public void testPictures() throws IOException, URISyntaxException { + ListalRipper ripper = + new ListalRipper(new URI("https://www.listal.com/emma-stone_iii/pictures").toURL()); + testRipper(ripper); + } + + /** + * Test for list type url. + */ + @Test + public void testRipListType() throws IOException, URISyntaxException { + ListalRipper ripper = + new ListalRipper(new URI("https://www.listal.com/list/evolution-emma-stone").toURL()); + testRipper(ripper); + } + + /** + * Test for folder type url. + */ + @Test + public void testRipFolderType() throws IOException, URISyntaxException { + ListalRipper ripper = + new ListalRipper(new URI("https://www.listal.com/chet-atkins/pictures").toURL()); + testRipper(ripper); + } + +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java index f2b7461b..44bf06cf 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java @@ -1,55 +1,55 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import com.rarchives.ripme.ripper.rippers.ScrolllerRipper; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.HashMap; -import java.util.Map; - -public class ScrolllerRipperTest extends RippersTest { - @Test - public void testScrolllerGID() throws IOException, URISyntaxException { - Map testURLs = new HashMap<>(); - - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp").toURL(), "CatsStandingUp"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures").toURL(), "CatsStandingUp"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=pictures").toURL(), "CatsStandingUp"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures&sort=top").toURL(), "CatsStandingUp"); - for (URL url : testURLs.keySet()) { - ScrolllerRipper ripper = new ScrolllerRipper(url); - ripper.setup(); - Assertions.assertEquals(testURLs.get(url), ripper.getGID(ripper.getURL())); - deleteDir(ripper.getWorkingDir()); - } - } - - @Test - public void testScrolllerFilterRegex() throws IOException, URISyntaxException { - Map testURLs = new HashMap<>(); - - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp").toURL(), "NOFILTER"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures").toURL(), "PICTURE"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=videos").toURL(), "VIDEO"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=albums").toURL(), "ALBUM"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=pictures").toURL(), "PICTURE"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=videos").toURL(), "VIDEO"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=albums").toURL(), "ALBUM"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures&sort=top").toURL(), "PICTURE"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=videos&sort=top").toURL(), "VIDEO"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=albums&sort=top").toURL(), "ALBUM"); - for (URL url : testURLs.keySet()) { - ScrolllerRipper ripper = new ScrolllerRipper(url); - ripper.setup(); - Assertions.assertEquals(testURLs.get(url), ripper.convertFilterString(ripper.getParameter(ripper.getURL(),"filter"))); - deleteDir(ripper.getWorkingDir()); - } - } - - - -} +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.ScrolllerRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; + +public class ScrolllerRipperTest extends RippersTest { + @Test + public void testScrolllerGID() throws IOException, URISyntaxException { + Map testURLs = new HashMap<>(); + + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp").toURL(), "CatsStandingUp"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures").toURL(), "CatsStandingUp"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=pictures").toURL(), "CatsStandingUp"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures&sort=top").toURL(), "CatsStandingUp"); + for (URL url : testURLs.keySet()) { + ScrolllerRipper ripper = new ScrolllerRipper(url); + ripper.setup(); + Assertions.assertEquals(testURLs.get(url), ripper.getGID(ripper.getURL())); + deleteDir(ripper.getWorkingDir()); + } + } + + @Test + public void testScrolllerFilterRegex() throws IOException, URISyntaxException { + Map testURLs = new HashMap<>(); + + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp").toURL(), "NOFILTER"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures").toURL(), "PICTURE"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=videos").toURL(), "VIDEO"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=albums").toURL(), "ALBUM"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=pictures").toURL(), "PICTURE"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=videos").toURL(), "VIDEO"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=albums").toURL(), "ALBUM"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures&sort=top").toURL(), "PICTURE"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=videos&sort=top").toURL(), "VIDEO"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=albums&sort=top").toURL(), "ALBUM"); + for (URL url : testURLs.keySet()) { + ScrolllerRipper ripper = new ScrolllerRipper(url); + ripper.setup(); + Assertions.assertEquals(testURLs.get(url), ripper.convertFilterString(ripper.getParameter(ripper.getURL(),"filter"))); + deleteDir(ripper.getWorkingDir()); + } + } + + + +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java index c6ab6a79..20e14442 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java @@ -1,52 +1,52 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import com.rarchives.ripme.ripper.rippers.VscoRipper; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; - -public class VscoRipperTest extends RippersTest { - - /** - * Testing single image. - * - * @throws IOException - */ - @Test - public void testSingleImageRip() throws IOException, URISyntaxException { - VscoRipper ripper = new VscoRipper(new URI("https://vsco.co/jolly-roger/media/597ce449846079297b3f7cf3").toURL()); - testRipper(ripper); - } - - /** - * Tests profile rip., Prevents Bug #679 from happening again. - * https://github.com/RipMeApp/ripme/issues/679 - * - * @throws IOException - */ - @Test - public void testHyphenatedRip() throws IOException, URISyntaxException { - VscoRipper ripper = new VscoRipper(new URI("https://vsco.co/jolly-roger/gallery").toURL()); - testRipper(ripper); - } - - /** - * Make sure it names the folder something sensible. - * - * @throws IOException - */ - @Test - public void testGetGID() throws IOException, URISyntaxException { - URL url = new URI("https://vsco.co/jolly-roger/media/590359c4ade3041f2658f407").toURL(); - - VscoRipper ripper = new VscoRipper(url); - - Assertions.assertEquals("jolly-roger/59035", ripper.getGID(url), "Failed to get GID"); - } - -} +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.VscoRipper; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; + +public class VscoRipperTest extends RippersTest { + + /** + * Testing single image. + * + * @throws IOException + */ + @Test + public void testSingleImageRip() throws IOException, URISyntaxException { + VscoRipper ripper = new VscoRipper(new URI("https://vsco.co/jolly-roger/media/597ce449846079297b3f7cf3").toURL()); + testRipper(ripper); + } + + /** + * Tests profile rip., Prevents Bug #679 from happening again. + * https://github.com/RipMeApp/ripme/issues/679 + * + * @throws IOException + */ + @Test + public void testHyphenatedRip() throws IOException, URISyntaxException { + VscoRipper ripper = new VscoRipper(new URI("https://vsco.co/jolly-roger/gallery").toURL()); + testRipper(ripper); + } + + /** + * Make sure it names the folder something sensible. + * + * @throws IOException + */ + @Test + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://vsco.co/jolly-roger/media/590359c4ade3041f2658f407").toURL(); + + VscoRipper ripper = new VscoRipper(url); + + Assertions.assertEquals("jolly-roger/59035", ripper.getGID(url), "Failed to get GID"); + } + +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XlecxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XlecxRipperTest.java index 2b943c7d..78eb5a3a 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XlecxRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XlecxRipperTest.java @@ -1,19 +1,19 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - -import com.rarchives.ripme.ripper.rippers.XlecxRipper; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -public class XlecxRipperTest extends RippersTest { - @Test - @Disabled("Broken ripper") - public void testAlbum() throws IOException, URISyntaxException { - XlecxRipper ripper = new XlecxRipper(new URI("http://xlecx.com/4274-black-canary-ravished-prey.html").toURL()); - testRipper(ripper); - } -} +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +import com.rarchives.ripme.ripper.rippers.XlecxRipper; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +public class XlecxRipperTest extends RippersTest { + @Test + @Disabled("Broken ripper") + public void testAlbum() throws IOException, URISyntaxException { + XlecxRipper ripper = new XlecxRipper(new URI("http://xlecx.com/4274-black-canary-ravished-prey.html").toURL()); + testRipper(ripper); + } +} diff --git a/utils/style.sh b/utils/style.sh deleted file mode 100644 index 45bb40e9..00000000 --- a/utils/style.sh +++ /dev/null @@ -1,27 +0,0 @@ -echo "" -echo "=====================================================" -echo "Tabs are not allowed" -echo "-----------------------------------------------------" -git grep -n -P "\t" -- :/*.java | sed -e "s/\t/\x1b[7m--->\x1b[m/g" -echo "=====================================================" - -echo "" -echo "=====================================================" -echo "Trailing whitespace is not allowed" -echo "-----------------------------------------------------" -git grep -n -P "[ \t]+$" -- :/*.java | sed -e "s/\t/\x1b[7m--->\x1b[m/g" | sed -e "s/ /\x1b[7m.\x1b[m/g" | sed -e "s/$/\x1b[7m$\x1b[m/g" -echo "=====================================================" - -echo "" -echo "=====================================================" -echo "'){' is not allowed. Place a space between ')' and '{', i.e. 'if (a) {'" -echo "-----------------------------------------------------" -git grep -n -P "\)\{" -- :/*.java -echo "=====================================================" - -echo "" -echo "=====================================================" -echo "A space is required after keywords (if|else|for|while|do|try|catch|finally)" -echo "-----------------------------------------------------" -git grep -n -P "(\b(if|for|while|catch)\b[(])|(\b(else|do|try|finally)\b[{])" -- :/*.java | sed -r -e "s/(\b(if|for|while|catch)\b[(])|(\b(else|do|try|finally)\b[{])/\x1b[7m\0\x1b[m/g" -echo "=====================================================" diff --git a/utils/stylefix.sh b/utils/stylefix.sh deleted file mode 100644 index dbfad1e1..00000000 --- a/utils/stylefix.sh +++ /dev/null @@ -1,17 +0,0 @@ -echo "" -echo "=====================================================" -echo "Tabs are not allowed (please manually fix tabs)" -echo "-----------------------------------------------------" -git grep -n -P "\t" -- :/*.java | sed -e "s/\t/\x1b[7m--->\x1b[m/g" -echo "=====================================================" - -echo "Removing trailing whitespace..." -git grep -l -P "[ \t]+$" -- :/*.java | xargs -I % sed -i -r -e "s/[ \t]+$//g" % - -echo "Replacing '){' with ') {'..." -git grep -l -P "\)\{" -- :/*.java | xargs -I % sed -i -r -e "s/\)\{/) {/g" % - -echo "Adding space between keywords and punctuation..." -git grep -l -P "(\b(if|for|while|catch)\b[(])" -- :/*.java | xargs -I % sed -i -r -e "s/(\b(if|for|while|catch)\b[(])/\2 (/g" % -git grep -l -P "(\b(else|do|try|finally)\b[{])" -- :/*.java | xargs -I % sed -i -r -e "s/(\b(else|do|try|finally)\b[{])/\2 {/g" % -