diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java index 66455861..bc824769 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -50,7 +52,7 @@ public class ArtStationRipper extends AbstractJSONRipper { try { // groupData = Http.url(albumURL.getLocation()).getJSON(); groupData = getJson(albumURL.getLocation()); - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { throw new MalformedURLException("Couldn't load JSON from " + albumURL.getLocation()); } return groupData.getString("title"); @@ -62,7 +64,7 @@ public class ArtStationRipper extends AbstractJSONRipper { try { // groupData = Http.url(userInfoURL).getJSON(); groupData = getJson(userInfoURL); - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { throw new MalformedURLException("Couldn't load JSON from " + userInfoURL); } return groupData.getString("full_name"); @@ -74,7 +76,7 @@ public class ArtStationRipper extends AbstractJSONRipper { } @Override - protected JSONObject getFirstPage() throws IOException { + protected JSONObject getFirstPage() throws IOException, URISyntaxException { if (albumURL.getType() == URL_TYPE.SINGLE_PROJECT) { // URL points to JSON of a single project, just return it // return Http.url(albumURL.getLocation()).getJSON(); @@ -90,7 +92,7 @@ public class ArtStationRipper extends AbstractJSONRipper { if (albumContent.getInt("total_count") > 0) { // Get JSON of the first project and return it JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(0); - ParsedURL projectURL = parseURL(new URL(projectInfo.getString("permalink"))); + ParsedURL projectURL = parseURL(new URI(projectInfo.getString("permalink")).toURL()); // return Http.url(projectURL.getLocation()).getJSON(); return getJson(projectURL.getLocation()); } @@ -100,7 +102,7 @@ public class ArtStationRipper extends AbstractJSONRipper { } @Override - protected JSONObject getNextPage(JSONObject doc) throws IOException { + protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException { if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) { // Initialize the page number if it hasn't been initialized already if (projectPageNumber == null) { @@ -117,7 +119,7 @@ public class ArtStationRipper extends AbstractJSONRipper { projectIndex = 0; } - Integer currentProject = ((projectPageNumber - 1) * 50) + (projectIndex + 1); + int currentProject = ((projectPageNumber - 1) * 50) + (projectIndex + 1); // JSONObject albumContent = Http.url(albumURL.getLocation() + "?page=" + // projectPageNumber).getJSON(); JSONObject albumContent = getJson(albumURL.getLocation() + "?page=" + projectPageNumber); @@ -125,7 +127,7 @@ public class ArtStationRipper extends AbstractJSONRipper { if (albumContent.getInt("total_count") > currentProject) { // Get JSON of the next project and return it JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(projectIndex); - ParsedURL projectURL = parseURL(new URL(projectInfo.getString("permalink"))); + ParsedURL projectURL = parseURL(new URI(projectInfo.getString("permalink")).toURL()); projectIndex++; // return Http.url(projectURL.getLocation()).getJSON(); return getJson(projectURL.getLocation()); @@ -320,8 +322,8 @@ public class ArtStationRipper extends AbstractJSONRipper { throw new IOException("Error fetching json. Status code:" + status); } - private JSONObject getJson(String url) throws IOException { - return getJson(new URL(url)); + private JSONObject getJson(String url) throws IOException, URISyntaxException { + return getJson(new URI(url).toURL()); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java index d86f63b8..f1d41426 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java @@ -72,7 +72,7 @@ public class ChanRipper extends AbstractHTMLRipper { ); private ChanSite chanSite; - private Boolean generalChanSite = true; + private boolean generalChanSite = true; public ChanRipper(URL url) throws IOException { super(url); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java index da9a0044..81f09aa4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java @@ -94,13 +94,6 @@ public class EHentaiRipper extends AbstractHTMLRipper { + " Got: " + url); } - /** - * Attempts to get page, checks for IP ban, waits. - * - * @param url - * @return Page document - * @throws IOException If page loading errors, or if retries are exhausted - */ private Document getPageWithRetries(URL url) throws IOException { Document doc; int retries = 3; @@ -251,16 +244,16 @@ public class EHentaiRipper extends AbstractHTMLRipper { savePath += String.format("%03d_", index); } savePath += m.group(1); - addURLToDownload(new URL(imgsrc), Paths.get(savePath)); + addURLToDownload(new URI(imgsrc).toURL(), Paths.get(savePath)); } else { // Provide prefix and let the AbstractRipper "guess" the filename String prefix = ""; if (Utils.getConfigBoolean("download.save_order", true)) { prefix = String.format("%03d_", index); } - addURLToDownload(new URL(imgsrc), prefix); + addURLToDownload(new URI(imgsrc).toURL(), prefix); } - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { LOGGER.error("[!] Exception while loading/parsing " + this.url, e); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java index 3ad3f684..3035d746 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -85,8 +86,8 @@ public class EromeRipper extends AbstractHTMLRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return new URL(url.toExternalForm().replaceAll("https?://erome.com", "https://www.erome.com")); + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { + return new URI(url.toExternalForm().replaceAll("https?://erome.com", "https://www.erome.com")).toURL(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java index 79edab1c..bba284f1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java @@ -1,9 +1,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.MalformedURLException; -import java.net.URL; +import java.net.*; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -164,8 +162,8 @@ public class FivehundredpxRipper extends AbstractJSONRipper { } @Override - public JSONObject getFirstPage() throws IOException { - URL apiURL = new URL(baseURL + "&consumer_key=" + CONSUMER_KEY); + public JSONObject getFirstPage() throws IOException, URISyntaxException { + URL apiURL = new URI(baseURL + "&consumer_key=" + CONSUMER_KEY).toURL(); LOGGER.debug("apiURL: " + apiURL); JSONObject json = Http.url(apiURL).getJSON(); @@ -232,7 +230,7 @@ public class FivehundredpxRipper extends AbstractJSONRipper { } @Override - public JSONObject getNextPage(JSONObject json) throws IOException { + public JSONObject getNextPage(JSONObject json) throws IOException, URISyntaxException { if (isThisATest()) { return null; } @@ -249,9 +247,9 @@ public class FivehundredpxRipper extends AbstractJSONRipper { sleep(500); ++page; - URL apiURL = new URL(baseURL + URL apiURL = new URI(baseURL + "&page=" + page - + "&consumer_key=" + CONSUMER_KEY); + + "&consumer_key=" + CONSUMER_KEY).toURL(); return Http.url(apiURL).getJSON(); } @@ -296,14 +294,9 @@ public class FivehundredpxRipper extends AbstractJSONRipper { } } } - if (imageURL == null) { - LOGGER.error("Failed to find image for photo " + photo.toString()); - } - else { - imageURLs.add(imageURL); - if (isThisATest()) { - break; - } + imageURLs.add(imageURL); + if (isThisATest()) { + break; } } return imageURLs; @@ -311,13 +304,13 @@ public class FivehundredpxRipper extends AbstractJSONRipper { private boolean urlExists(String url) { try { - HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); + HttpURLConnection connection = (HttpURLConnection) new URI(url).toURL().openConnection(); connection.setRequestMethod("HEAD"); if (connection.getResponseCode() != 200) { throw new IOException("Couldn't find full-size image at " + url); } return true; - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { return false; } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java index d88b16e8..62a60fcc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -40,7 +42,7 @@ public class FuskatorRipper extends AbstractHTMLRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { String u = url.toExternalForm(); if (u.contains("/thumbs/")) { u = u.replace("/thumbs/", "/full/"); @@ -48,7 +50,7 @@ public class FuskatorRipper extends AbstractHTMLRipper { if (u.contains("/expanded/")) { u = u.replaceAll("/expanded/", "/full/"); } - return new URL(u); + return new URI(u).toURL(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java index ca709418..4d28f7a2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.Base64; @@ -80,16 +82,15 @@ public class HentaiNexusRipper extends AbstractJSONRipper { } @Override - protected JSONObject getFirstPage() throws IOException { + protected JSONObject getFirstPage() throws IOException, URISyntaxException { String jsonEncodedString = getJsonEncodedStringFromPage(); String jsonDecodedString = decodeJsonString(jsonEncodedString); return new JSONObject(jsonDecodedString); } - public String getJsonEncodedStringFromPage() throws MalformedURLException, IOException - { + public String getJsonEncodedStringFromPage() throws MalformedURLException, IOException, URISyntaxException { // Image data only appears on the /read/ page and not on the /view/ one. - URL readUrl = new URL(String.format("http://hentainexus.com/read/%s",getGID(url))); + URL readUrl = new URI(String.format("http://hentainexus.com/read/%s",getGID(url))).toURL(); Document document = Http.url(readUrl).response().parse(); for (Element scripts : document.getElementsByTag("script")) { @@ -143,7 +144,7 @@ public class HentaiNexusRipper extends AbstractJSONRipper { } magicByte = (byte) (magicByte & 0x7); - ArrayList newArray = new ArrayList(); + ArrayList newArray = new ArrayList<>(); for (int i = 0x0; i < 0x100; i++) { newArray.add(i); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java index d7c46a97..d312b75b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -14,7 +15,6 @@ import org.jsoup.nodes.Document; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; -import org.jsoup.nodes.Element; public class HitomiRipper extends AbstractHTMLRipper { @@ -47,9 +47,9 @@ public class HitomiRipper extends AbstractHTMLRipper { } @Override - public Document getFirstPage() throws IOException { + public Document getFirstPage() throws IOException, URISyntaxException { // if we go to /GALLERYID.js we get a nice json array of all images in the gallery - return Http.url(new URL(url.toExternalForm().replaceAll("hitomi", "ltn.hitomi").replaceAll(".html", ".js"))).ignoreContentType().get(); + return Http.url(new URI(url.toExternalForm().replaceAll("hitomi", "ltn.hitomi").replaceAll(".html", ".js")).toURL()).ignoreContentType().get(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java index 30da2344..7e26faa2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java @@ -8,6 +8,8 @@ import org.json.JSONObject; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -34,7 +36,7 @@ public class NsfwXxxRipper extends AbstractJSONRipper { @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { String u = url.toExternalForm(); // https://nsfw.xxx/user/kelly-kat/foo -> https://nsfw.xxx/user/kelly-kat // https://nsfw.xxx/user/kelly-kat -> https://nsfw.xxx/user/kelly-kat @@ -44,15 +46,15 @@ public class NsfwXxxRipper extends AbstractJSONRipper { throw new MalformedURLException("Invalid URL: " + url); } - return new URL(u); + return new URI(u).toURL(); } String getUser() throws MalformedURLException { return getGID(url); } - URL getPage(int page) throws MalformedURLException { - return new URL("https://nsfw.xxx/slide-page/" + page + "?nsfw%5B%5D=0&types%5B%5D=image&types%5B%5D=video&types%5B%5D=gallery&slider=1&jsload=1&user=" + getUser()); + URL getPage(int page) throws MalformedURLException, URISyntaxException { + return new URI("https://nsfw.xxx/slide-page/" + page + "?nsfw%5B%5D=0&types%5B%5D=image&types%5B%5D=video&types%5B%5D=gallery&slider=1&jsload=1&user=" + getUser()).toURL(); } @@ -71,18 +73,18 @@ public class NsfwXxxRipper extends AbstractJSONRipper { int currentPage = 1; @Override - protected JSONObject getFirstPage() throws IOException { + protected JSONObject getFirstPage() throws IOException, URISyntaxException { return Http.url(getPage(1)).getJSON(); } List descriptions = new ArrayList<>(); @Override - protected JSONObject getNextPage(JSONObject doc) throws IOException { + protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException { currentPage++; JSONObject nextPage = Http.url(getPage(doc.getInt("page") + 1)).getJSON(); JSONArray items = nextPage.getJSONArray("items"); - if (items.length() == 0) { + if (items.isEmpty()) { throw new IOException("No more pages"); } return nextPage; @@ -120,7 +122,7 @@ public class NsfwXxxRipper extends AbstractJSONRipper { return new ApiEntry(srcUrl, o.getString("author"), o.getString("title")); }) - .collect(Collectors.toList()); + .toList(); data.forEach(e -> descriptions.add(e.title)); return data.stream().map(e -> e.srcUrl).collect(Collectors.toList()); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 4c62a184..2cea95a7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -44,13 +45,13 @@ public class XhamsterRipper extends AbstractHTMLRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { if (isVideoUrl(url)) { return url; } String URLToReturn = url.toExternalForm(); URLToReturn = URLToReturn.replaceAll("https?://\\w?\\w?\\.?xhamster([^<]*)\\.", "https://m.xhamster$1."); - URL san_url = new URL(URLToReturn); + URL san_url = new URI(URLToReturn).toURL(); LOGGER.info("sanitized URL is " + san_url.toExternalForm()); return san_url; } @@ -168,10 +169,10 @@ public class XhamsterRipper extends AbstractHTMLRipper { // This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to // the page chamster.com but displays the mobile site from m.xhamster.com pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster([^<]*)\\.", "://m.xhamster$1."); - String image = Http.url(new URL(pageWithImageUrl)).get().select("a > img#photoCurr").attr("src"); + String image = Http.url(new URI(pageWithImageUrl).toURL()).get().select("a > img#photoCurr").attr("src"); result.add(image); downloadFile(image); - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { LOGGER.error("Was unable to load page " + pageWithImageUrl); } if (isStopped() || isThisATest()) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java index 16526945..a9c39a9c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -54,7 +56,7 @@ public class CliphunterRipper extends VideoRipper { } @Override - public void rip() throws IOException { + public void rip() throws IOException, URISyntaxException { LOGGER.info("Retrieving " + this.url); String html = Http.url(url).get().html(); String jsonString = html.substring(html.indexOf("var flashVars = {d: '") + 21); @@ -71,7 +73,7 @@ public class CliphunterRipper extends VideoRipper { vidURL += c; } } - addURLToDownload(new URL(vidURL), HOST + "_" + getGID(this.url)); + addURLToDownload(new URI(vidURL).toURL(), HOST + "_" + getGID(this.url)); waitForThreads(); } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java index 678435af..5a3dcebb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; @@ -56,7 +58,7 @@ public class PornhubRipper extends VideoRipper { } @Override - public void rip() throws IOException { + public void rip() throws IOException, URISyntaxException { String vidUrl = ""; LOGGER.info(" Retrieving " + this.url.toExternalForm()); Document doc = Http.url(this.url).get(); @@ -146,7 +148,7 @@ public class PornhubRipper extends VideoRipper { if (vidUrl.equals("")) { throw new IOException("Unable to find encrypted video URL at " + this.url); } - addURLToDownload(new URL(vidUrl), HOST + "_" + bestQuality + "p_" + getGID(this.url)); + addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + bestQuality + "p_" + getGID(this.url)); waitForThreads(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/StickyXXXRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/StickyXXXRipper.java index 7c951b23..8708e552 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/StickyXXXRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/StickyXXXRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -32,11 +34,6 @@ public class StickyXXXRipper extends VideoRipper { return m.matches(); } - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return url; - } - @Override public String getGID(URL url) throws MalformedURLException { Pattern p = Pattern.compile("^https?://.*stickyxxx\\.com(/)(.*)/$"); @@ -52,7 +49,7 @@ public class StickyXXXRipper extends VideoRipper { } @Override - public void rip() throws IOException { + public void rip() throws IOException, URISyntaxException { LOGGER.info("Retrieving " + this.url); Document doc = Http.url(url).get(); Elements videos = doc.select(".wp-video > video > source"); @@ -60,7 +57,7 @@ public class StickyXXXRipper extends VideoRipper { throw new IOException("Could not find Embed code at " + url); } String vidUrl = videos.attr("src"); - addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url)); + addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); waitForThreads(); } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java index c72a5a59..bd4ee556 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -34,11 +36,6 @@ public class TwitchVideoRipper extends VideoRipper { return m.matches(); } - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return url; - } - @Override public String getGID(URL url) throws MalformedURLException { Pattern p = Pattern.compile("^https://clips\\.twitch\\.tv/(.*)$"); @@ -54,7 +51,7 @@ public class TwitchVideoRipper extends VideoRipper { } @Override - public void rip() throws IOException { + public void rip() throws IOException, URISyntaxException { LOGGER.info("Retrieving " + this.url); Document doc = Http.url(url).get(); @@ -72,7 +69,7 @@ public class TwitchVideoRipper extends VideoRipper { Matcher m = p.matcher(element.data()); if (m.find()){ String vidUrl = m.group(1); - addURLToDownload(new URL(vidUrl), HOST + "_" + title); + addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + title); } } waitForThreads(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java index 078b32a5..279e1d3a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -52,7 +54,7 @@ public class ViddmeRipper extends VideoRipper { } @Override - public void rip() throws IOException { + public void rip() throws IOException, URISyntaxException { LOGGER.info(" Retrieving " + this.url.toExternalForm()); Document doc = Http.url(this.url).get(); Elements videos = doc.select("meta[name=twitter:player:stream]"); @@ -61,7 +63,7 @@ public class ViddmeRipper extends VideoRipper { } String vidUrl = videos.first().attr("content"); vidUrl = vidUrl.replaceAll("&", "&"); - addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url)); + addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); waitForThreads(); } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java index 052b2cbe..3fbb6375 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.List; import java.util.regex.Matcher; @@ -33,11 +35,6 @@ public class VidearnRipper extends VideoRipper { return m.matches(); } - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return url; - } - @Override public String getGID(URL url) throws MalformedURLException { Pattern p = Pattern.compile("^https?://[wm.]*videarn\\.com/[a-zA-Z0-9\\-]+/([0-9]+).*$"); @@ -53,15 +50,15 @@ public class VidearnRipper extends VideoRipper { } @Override - public void rip() throws IOException { + public void rip() throws IOException, URISyntaxException { LOGGER.info("Retrieving " + this.url); Document doc = Http.url(url).get(); List mp4s = Utils.between(doc.html(), "file:\"", "\""); if (mp4s.isEmpty()) { throw new IOException("Could not find files at " + url); } - String vidUrl = mp4s.get(0); - addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url)); + String vidUrl = mp4s.getFirst(); + addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); waitForThreads(); } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java index 70528727..84206abb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -51,10 +53,10 @@ public class VkRipper extends VideoRipper { } @Override - public void rip() throws IOException { + public void rip() throws IOException, URISyntaxException { LOGGER.info(" Retrieving " + this.url); String videoURL = getVideoURLAtPage(this.url.toExternalForm()); - addURLToDownload(new URL(videoURL), HOST + "_" + getGID(this.url)); + addURLToDownload(new URI(videoURL).toURL(), HOST + "_" + getGID(this.url)); waitForThreads(); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StickyXXXRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StickyXXXRipperTest.java index 57a07d90..71038c94 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StickyXXXRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StickyXXXRipperTest.java @@ -7,9 +7,11 @@ import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.video.StickyXXXRipper; // import com.rarchives.ripme.tst.ripper.rippers.RippersTest; import com.rarchives.ripme.utils.Utils; +import org.junit.jupiter.api.Test; public class StickyXXXRipperTest extends RippersTest { + @Test public void testStickyXXXVideo() throws IOException, URISyntaxException { // This test fails on the CI - possibly due to checking for a file before it's written - so we're skipping it if (Utils.getConfigBoolean("test.run_flaky_tests", false)) {