From dd6ac9bc125dd955a650355da32e2cb34fb515df Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 27 Oct 2023 21:59:28 +0200 Subject: [PATCH] new URL deprecated, use URI --- .../ripme/ripper/AbstractHTMLRipper.java | 3 +- .../ripme/ripper/DownloadFileThread.java | 9 ++-- .../ripper/rippers/EightmusesRipper.java | 6 ++- .../ripme/ripper/rippers/EroShareRipper.java | 8 ++-- .../ripme/ripper/rippers/ErofusRipper.java | 6 ++- .../ripme/ripper/rippers/ImagefapRipper.java | 14 +++--- .../ripper/rippers/JagodibujaRipper.java | 6 ++- .../ripme/ripper/rippers/LusciousRipper.java | 7 +-- .../ripme/ripper/rippers/MultpornRipper.java | 3 +- .../ripme/ripper/rippers/RedgifsRipper.java | 46 +++++++++++-------- .../ripme/ripper/rippers/TapasticRipper.java | 6 ++- .../ripme/ripper/rippers/TsuminoRipper.java | 6 ++- .../ripme/ripper/rippers/VidbleRipper.java | 6 ++- .../com/rarchives/ripme/utils/RipUtils.java | 4 +- 14 files changed, 76 insertions(+), 54 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index f087f980..6380a1c6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -3,6 +3,7 @@ package com.rarchives.ripme.ripper; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.*; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -54,7 +55,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { public Document getNextPage(Document doc) throws IOException, URISyntaxException { return null; } - protected abstract List getURLsFromPage(Document page); + protected abstract List getURLsFromPage(Document page) throws UnsupportedEncodingException; protected List getDescriptionsFromPage(Document doc) throws IOException { throw new IOException("getDescriptionsFromPage not implemented"); // Do I do this or make an abstract function? } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index a6722971..e9c6f242 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -1,10 +1,7 @@ package com.rarchives.ripme.ripper; import java.io.*; -import java.net.HttpURLConnection; -import java.net.SocketTimeoutException; -import java.net.URL; -import java.net.URLConnection; +import java.net.*; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Arrays; @@ -152,7 +149,7 @@ class DownloadFileThread implements Runnable { redirected = true; } String location = huc.getHeaderField("Location"); - urlToDownload = new URL(location); + urlToDownload = new URI(location).toURL(); // Throw exception so download can be retried throw new IOException("Redirect status code " + statusCode + " - redirect to " + location); } @@ -284,7 +281,7 @@ class DownloadFileThread implements Runnable { "HTTP status code " + hse.getStatusCode() + " while downloading " + url.toExternalForm()); return; } - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { logger.debug("IOException", e); logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + e.getMessage()); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java index c9984569..f60ced28 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; @@ -114,7 +116,7 @@ public class EightmusesRipper extends AbstractHTMLRipper { try { for (int i = 0; i != json.getJSONArray("pictures").length(); i++) { image = "https://www.8muses.com/image/fl/" + json.getJSONArray("pictures").getJSONObject(i).getString("publicUri"); - URL imageUrl = new URL(image); + URL imageUrl = new URI(image).toURL(); addURLToDownload(imageUrl, getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, getPrefixShort(x), "", null, true); // X is our page index x++; @@ -123,7 +125,7 @@ public class EightmusesRipper extends AbstractHTMLRipper { } } return imageURLs; - } catch (MalformedURLException e) { + } catch (MalformedURLException | URISyntaxException e) { LOGGER.error("\"" + image + "\" is malformed"); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java index bcdca5e4..d7b8015a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java @@ -7,6 +7,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -194,7 +196,7 @@ public class EroShareRipper extends AbstractHTMLRipper { throw new MalformedURLException("eroshare album not found in " + url + ", expected https://eroshare.com/album or eroshae.com/album"); } - public static List getURLs(URL url) throws IOException{ + public static List getURLs(URL url) throws IOException, URISyntaxException { Response resp = Http.url(url) .ignoreContentType() @@ -208,7 +210,7 @@ public class EroShareRipper extends AbstractHTMLRipper { for (Element img : imgs) { if (img.hasClass("album-image")) { String imageURL = img.attr("src"); - URLs.add(new URL(imageURL)); + URLs.add(new URI(imageURL).toURL()); } } //Videos @@ -217,7 +219,7 @@ public class EroShareRipper extends AbstractHTMLRipper { if (vid.hasClass("album-video")) { Elements source = vid.getElementsByTag("source"); String videoURL = source.first().attr("src"); - URLs.add(new URL(videoURL)); + URLs.add(new URI(videoURL).toURL()); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java index cd95d7fc..95528470 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java @@ -9,6 +9,8 @@ import org.jsoup.select.Elements; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; @@ -89,8 +91,8 @@ public class ErofusRipper extends AbstractHTMLRipper { Map opts = new HashMap(); opts.put("subdirectory", page.title().replaceAll(" \\| Erofus - Sex and Porn Comics", "").replaceAll(" ", "_")); opts.put("prefix", getPrefix(x)); - addURLToDownload(new URL(image), opts); - } catch (MalformedURLException e) { + addURLToDownload(new URI(image).toURL(), opts); + } catch (MalformedURLException | URISyntaxException e) { LOGGER.info(e.getMessage()); } x++; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java index 5ef89fbd..bcd5900f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java @@ -4,6 +4,8 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.text.MessageFormat; import java.util.ArrayList; @@ -51,11 +53,11 @@ public class ImagefapRipper extends AbstractHTMLRipper { * Reformat given URL into the desired format (all images on single page) */ @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { String gid = getGID(url); String newURL = "https://www.imagefap.com/pictures/" + gid + "/random-string"; LOGGER.debug("Changed URL from " + url + " to " + newURL); - return new URL(newURL); + return new URI(newURL).toURL(); } @Override @@ -106,7 +108,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { } @Override - public Document getNextPage(Document doc) throws IOException { + public Document getNextPage(Document doc) throws IOException, URISyntaxException { String nextURL = null; for (Element a : doc.select("a.link3")) { if (a.text().contains("next")) { @@ -124,7 +126,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { LOGGER.info("Attempting to load next page URL: " + nextURL); // Load next page - Document nextPage = getPageWithRetries(new URL(nextURL)); + Document nextPage = getPageWithRetries(new URI(nextURL).toURL()); return nextPage; } @@ -182,7 +184,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { // Sleep before fetching image. sleep(IMAGE_SLEEP_TIME); - Document doc = getPageWithRetries(new URL(pageURL)); + Document doc = getPageWithRetries(new URI(pageURL).toURL()); String framedPhotoUrl = doc.select("img#mainPhoto").attr("data-src"); @@ -204,7 +206,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { return fullSizedUrl; - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { LOGGER.debug("Unable to get full size image URL from page: " + pageURL + " because: " + e.getMessage()); return null; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/JagodibujaRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/JagodibujaRipper.java index 2c16b2e5..2f2d5c33 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/JagodibujaRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/JagodibujaRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -56,8 +58,8 @@ public class JagodibujaRipper extends AbstractHTMLRipper { Element elem = comicPage.select("span.full-size-link > a").first(); LOGGER.info("Got link " + elem.attr("href")); try { - addURLToDownload(new URL(elem.attr("href")), ""); - } catch (MalformedURLException e) { + addURLToDownload(new URI(elem.attr("href")).toURL(), ""); + } catch (MalformedURLException | URISyntaxException e) { LOGGER.warn("Malformed URL"); e.printStackTrace(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java index 930eb38d..099eaf73 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java @@ -9,6 +9,7 @@ import org.jsoup.select.Elements; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -95,7 +96,7 @@ public class LusciousRipper extends AbstractHTMLRipper { sanitizedUrl = sanitizedUrl.replaceFirst( "^https?://(?:members\\.|legacy\\.|www\\.)?luscious.net", "https://legacy.luscious.net"); - return new URL(sanitizedUrl); + return new URI(sanitizedUrl).toURL(); } throw new Exception("ERROR: Unable to sanitize url."); @@ -142,9 +143,9 @@ public class LusciousRipper extends AbstractHTMLRipper { } //If a valid download url was found. - addURLToDownload(new URL(downloadUrl), getPrefix(index)); + addURLToDownload(new URI(downloadUrl).toURL(), getPrefix(index)); - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { LOGGER.error("Error downloadiong url " + url, e); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java index 42683694..cdc873f2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java @@ -8,6 +8,7 @@ import org.jsoup.select.Elements; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -44,7 +45,7 @@ public class MultpornRipper extends AbstractHTMLRipper { p = Pattern.compile("/node/(\\d+)/.*"); m = p.matcher(nodeHref); if (m.matches()) { - this.url = new URL("https://multporn.net" + nodeHref); + this.url = new URI("https://multporn.net" + nodeHref).toURL(); return m.group(1); } }catch (Exception ignored){}; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java index 3c8547a9..472d6d3a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java @@ -11,6 +11,8 @@ import org.jsoup.select.Elements; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -29,8 +31,8 @@ public class RedgifsRipper extends AbstractHTMLRipper { int searchCount = 150; int searchStart = 0; - public RedgifsRipper(URL url) throws IOException { - super(new URL(url.toExternalForm().replace("thumbs.", ""))); + public RedgifsRipper(URL url) throws IOException, URISyntaxException { + super(new URI(url.toExternalForm().replace("thumbs.", "")).toURL()); } @Override @@ -47,12 +49,12 @@ public class RedgifsRipper extends AbstractHTMLRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { String sUrl = url.toExternalForm(); sUrl = sUrl.replace("/gifs/detail", ""); sUrl = sUrl.replace("/amp", ""); sUrl = sUrl.replace("gifdeliverynetwork.com", "redgifs.com/watch"); - return new URL(sUrl); + return new URI(sUrl).toURL(); } public Matcher isProfile() { @@ -72,16 +74,20 @@ public class RedgifsRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - if (!isProfile().matches() && !isSearch().matches()) { - return Http.url(url).get(); - } else if (isSearch().matches()) { - searchText = getGID(url).replace("-", " "); - return Http.url( - new URL("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart*searchCount)).ignoreContentType().get(); - } else { - username = getGID(url); - return Http.url(new URL("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count)) - .ignoreContentType().get(); + try { + if (!isProfile().matches() && !isSearch().matches()) { + return Http.url(url).get(); + } else if (isSearch().matches()) { + searchText = getGID(url).replace("-", " "); + return Http.url( + new URI("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart * searchCount).toURL()).ignoreContentType().get(); + } else { + username = getGID(url); + return Http.url(new URI("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count).toURL()) + .ignoreContentType().get(); + } + } catch (URISyntaxException e) { + throw new IOException(e); } } @@ -124,18 +130,18 @@ public class RedgifsRipper extends AbstractHTMLRipper { } @Override - public Document getNextPage(Document doc) throws IOException { + public Document getNextPage(Document doc) throws IOException, URISyntaxException { if (isSearch().matches()) { Document d = Http.url( - new URL("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText - + "&count=" + searchCount + "&start=" + searchCount*++searchStart)) + new URI("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + + "&count=" + searchCount + "&start=" + searchCount*++searchStart).toURL()) .ignoreContentType().get(); return (hasURLs(d).isEmpty()) ? null : d; } else { if (cursor.equals("") || cursor.equals("null")) { return null; } else { - Document d = Http.url(new URL("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor)).ignoreContentType().get(); + Document d = Http.url(new URI("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor).toURL()).ignoreContentType().get(); return (hasURLs(d).isEmpty()) ? null : d; } } @@ -182,11 +188,11 @@ public class RedgifsRipper extends AbstractHTMLRipper { * @return URL to video * @throws IOException */ - public static String getVideoURL(URL url) throws IOException { + public static String getVideoURL(URL url) throws IOException, URISyntaxException { LOGGER.info("Retrieving " + url.toExternalForm()); //Sanitize the URL first - url = new URL(url.toExternalForm().replace("/gifs/detail", "")); + url = new URI(url.toExternalForm().replace("/gifs/detail", "")).toURL(); Document doc = Http.url(url).get(); Elements videos = doc.select("script"); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java index d4a0d8f3..d514c1e6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -82,12 +84,12 @@ public class TapasticRipper extends AbstractHTMLRipper { prefix.append(String.format("-%0" + imgLog + "dof%0" + imgLog + "d-", i + 1, images.size())); prefix.append(episode.filename.replace(" ", "-")); prefix.append("-"); - addURLToDownload(new URL(link), prefix.toString()); + addURLToDownload(new URI(link).toURL(), prefix.toString()); if (isThisATest()) { break; } } - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { LOGGER.error("[!] Exception while downloading " + url, e); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java index 31917199..49baa384 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java @@ -1,9 +1,11 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -110,11 +112,11 @@ public class TsuminoRipper extends AbstractHTMLRipper { } @Override - public List getURLsFromPage(Document doc) { + public List getURLsFromPage(Document doc) throws UnsupportedEncodingException { JSONArray imageIds = getPageUrls(); List result = new ArrayList<>(); for (int i = 0; i < imageIds.length(); i++) { - result.add("http://www.tsumino.com/Image/Object?name=" + URLEncoder.encode(imageIds.getString(i))); + result.add("http://www.tsumino.com/Image/Object?name=" + URLEncoder.encode(imageIds.getString(i), StandardCharsets.UTF_8.name())); } return result; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java index e3888f75..baf5e212 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -70,11 +72,11 @@ public class VidbleRipper extends AbstractHTMLRipper { addURLToDownload(url, getPrefix(index)); } - public static List getURLsFromPage(URL url) throws IOException { + public static List getURLsFromPage(URL url) throws IOException, URISyntaxException { List urls = new ArrayList<>(); Document doc = Http.url(url).get(); for (String stringURL : getURLsFromPageStatic(doc)) { - urls.add(new URL(stringURL)); + urls.add(new URI(stringURL).toURL()); } return urls; } diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 85d092e8..8067cd92 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -84,7 +84,7 @@ public class RipUtils { try { logger.info("Getting vidble album " + url); result.addAll(VidbleRipper.getURLsFromPage(url)); - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { // Do nothing logger.warn("Exception while retrieving vidble page:", e); } @@ -94,7 +94,7 @@ public class RipUtils { try { logger.info("Getting eroshare album " + url); result.addAll(EroShareRipper.getURLs(url)); - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { // Do nothing logger.warn("Exception while retrieving eroshare page:", e); }