new URL deprecated, use URI

src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java

@@ -3,6 +3,7 @@ package com.rarchives.ripme.ripper;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.net.*;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -54,7 +55,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
     public Document getNextPage(Document doc) throws IOException, URISyntaxException {
         return null;
     }
-    protected abstract List<String> getURLsFromPage(Document page);
+    protected abstract List<String> getURLsFromPage(Document page) throws UnsupportedEncodingException;
     protected List<String> getDescriptionsFromPage(Document doc) throws IOException {
         throw new IOException("getDescriptionsFromPage not implemented"); // Do I do this or make an abstract function?
     }

src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java

@@ -1,10 +1,7 @@
 package com.rarchives.ripme.ripper;
 
 import java.io.*;
-import java.net.HttpURLConnection;
+import java.net.*;
-import java.net.SocketTimeoutException;
-import java.net.URL;
-import java.net.URLConnection;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.Arrays;
@@ -152,7 +149,7 @@ class DownloadFileThread implements Runnable {
                         redirected = true;
                     }
                     String location = huc.getHeaderField("Location");
-                    urlToDownload = new URL(location);
+                    urlToDownload = new URI(location).toURL();
                     // Throw exception so download can be retried
                     throw new IOException("Redirect status code " + statusCode + " - redirect to " + location);
                 }
@@ -284,7 +281,7 @@ class DownloadFileThread implements Runnable {
                             "HTTP status code " + hse.getStatusCode() + " while downloading " + url.toExternalForm());
                     return;
                 }
-            } catch (IOException e) {
+            } catch (IOException | URISyntaxException e) {
                 logger.debug("IOException", e);
                 logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - "
                         + e.getMessage());

src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers;
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -114,7 +116,7 @@ public class EightmusesRipper extends AbstractHTMLRipper {
                     try {
                         for (int i = 0; i != json.getJSONArray("pictures").length(); i++) {
                             image = "https://www.8muses.com/image/fl/" + json.getJSONArray("pictures").getJSONObject(i).getString("publicUri");
-                            URL imageUrl = new URL(image);
+                            URL imageUrl = new URI(image).toURL();
                             addURLToDownload(imageUrl, getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, getPrefixShort(x), "", null, true);
                             // X is our page index
                             x++;
@@ -123,7 +125,7 @@ public class EightmusesRipper extends AbstractHTMLRipper {
                             }
                         }
                         return imageURLs;
-                    } catch (MalformedURLException e) {
+                    } catch (MalformedURLException | URISyntaxException e) {
                         LOGGER.error("\"" + image + "\" is malformed");
                     }
                 }

src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java

@@ -7,6 +7,8 @@ package com.rarchives.ripme.ripper.rippers;
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
@@ -194,7 +196,7 @@ public class EroShareRipper extends AbstractHTMLRipper {
         throw new MalformedURLException("eroshare album not found in " + url + ", expected https://eroshare.com/album or eroshae.com/album");
     }
 
-    public static List<URL> getURLs(URL url) throws IOException{
+    public static List<URL> getURLs(URL url) throws IOException, URISyntaxException {
 
         Response resp = Http.url(url)
                             .ignoreContentType()
@@ -208,7 +210,7 @@ public class EroShareRipper extends AbstractHTMLRipper {
         for (Element img : imgs) {
             if (img.hasClass("album-image")) {
                 String imageURL = img.attr("src");
-                URLs.add(new URL(imageURL));
+                URLs.add(new URI(imageURL).toURL());
             }
         }
         //Videos
@@ -217,7 +219,7 @@ public class EroShareRipper extends AbstractHTMLRipper {
             if (vid.hasClass("album-video")) {
                 Elements source = vid.getElementsByTag("source");
                 String videoURL = source.first().attr("src");
-                URLs.add(new URL(videoURL));
+                URLs.add(new URI(videoURL).toURL());
             }
         }
 

src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java

@@ -9,6 +9,8 @@ import org.jsoup.select.Elements;
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -89,8 +91,8 @@ public class ErofusRipper extends AbstractHTMLRipper {
                 Map<String,String> opts = new HashMap<String, String>();
                 opts.put("subdirectory", page.title().replaceAll(" \\| Erofus - Sex and Porn Comics", "").replaceAll(" ", "_"));
                 opts.put("prefix", getPrefix(x));
-                addURLToDownload(new URL(image), opts);
+                addURLToDownload(new URI(image).toURL(), opts);
-            } catch (MalformedURLException e) {
+            } catch (MalformedURLException | URISyntaxException e) {
                 LOGGER.info(e.getMessage());
             }
             x++;

src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java

@@ -4,6 +4,8 @@ import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.text.MessageFormat;
 import java.util.ArrayList;
@@ -51,11 +53,11 @@ public class ImagefapRipper extends AbstractHTMLRipper {
      * Reformat given URL into the desired format (all images on single page)
      */
     @Override
-    public URL sanitizeURL(URL url) throws MalformedURLException {
+    public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
         String gid = getGID(url);
         String newURL = "https://www.imagefap.com/pictures/" + gid + "/random-string";
         LOGGER.debug("Changed URL from " + url + " to " + newURL);
-        return new URL(newURL);
+        return new URI(newURL).toURL();
     }
 
     @Override
@@ -106,7 +108,7 @@ public class ImagefapRipper extends AbstractHTMLRipper {
     }
 
     @Override
-    public Document getNextPage(Document doc) throws IOException {
+    public Document getNextPage(Document doc) throws IOException, URISyntaxException {
         String nextURL = null;
         for (Element a : doc.select("a.link3")) {
             if (a.text().contains("next")) {
@@ -124,7 +126,7 @@ public class ImagefapRipper extends AbstractHTMLRipper {
         LOGGER.info("Attempting to load next page URL: " + nextURL);
 
         // Load next page
-        Document nextPage = getPageWithRetries(new URL(nextURL));
+        Document nextPage = getPageWithRetries(new URI(nextURL).toURL());
 
         return nextPage;
     }
@@ -182,7 +184,7 @@ public class ImagefapRipper extends AbstractHTMLRipper {
             // Sleep before fetching image.
             sleep(IMAGE_SLEEP_TIME);
 
-            Document doc = getPageWithRetries(new URL(pageURL));
+            Document doc = getPageWithRetries(new URI(pageURL).toURL());
 
             String framedPhotoUrl = doc.select("img#mainPhoto").attr("data-src");
 
@@ -204,7 +206,7 @@ public class ImagefapRipper extends AbstractHTMLRipper {
 
             return fullSizedUrl;
 
-        } catch (IOException e) {
+        } catch (IOException | URISyntaxException e) {
             LOGGER.debug("Unable to get full size image URL from page: " + pageURL + " because: " +  e.getMessage());
             return null;
         }

src/main/java/com/rarchives/ripme/ripper/rippers/JagodibujaRipper.java

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers;
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
@@ -56,8 +58,8 @@ public class JagodibujaRipper extends AbstractHTMLRipper {
                 Element elem = comicPage.select("span.full-size-link > a").first();
                 LOGGER.info("Got link " + elem.attr("href"));
                 try {
-                    addURLToDownload(new URL(elem.attr("href")), "");
+                    addURLToDownload(new URI(elem.attr("href")).toURL(), "");
-                } catch (MalformedURLException e) {
+                } catch (MalformedURLException | URISyntaxException e) {
                     LOGGER.warn("Malformed URL");
                     e.printStackTrace();
                 }

src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java

@@ -9,6 +9,7 @@ import org.jsoup.select.Elements;
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
@@ -95,7 +96,7 @@ public class LusciousRipper extends AbstractHTMLRipper {
                 sanitizedUrl = sanitizedUrl.replaceFirst(
                         "^https?://(?:members\\.|legacy\\.|www\\.)?luscious.net",
                         "https://legacy.luscious.net");
-                return new URL(sanitizedUrl);
+                return new URI(sanitizedUrl).toURL();
             }
 
             throw new Exception("ERROR: Unable to sanitize url.");
@@ -142,9 +143,9 @@ public class LusciousRipper extends AbstractHTMLRipper {
                 }
 
                 //If a valid download url was found.
-                addURLToDownload(new URL(downloadUrl), getPrefix(index));
+                addURLToDownload(new URI(downloadUrl).toURL(), getPrefix(index));
 
-            } catch (IOException e) {
+            } catch (IOException | URISyntaxException e) {
                 LOGGER.error("Error downloadiong url " + url, e);
             }
         }

src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java

@@ -8,6 +8,7 @@ import org.jsoup.select.Elements;
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
@@ -44,7 +45,7 @@ public class MultpornRipper extends AbstractHTMLRipper {
             p = Pattern.compile("/node/(\\d+)/.*");
             m = p.matcher(nodeHref);
             if (m.matches()) {
-                this.url = new URL("https://multporn.net" + nodeHref);
+                this.url = new URI("https://multporn.net" + nodeHref).toURL();
                 return m.group(1);
             }
         }catch (Exception ignored){};

src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java

@@ -11,6 +11,8 @@ import org.jsoup.select.Elements;
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
@@ -29,8 +31,8 @@ public class RedgifsRipper extends AbstractHTMLRipper {
     int searchCount = 150;
     int searchStart = 0;
 
-    public RedgifsRipper(URL url) throws IOException {
+    public RedgifsRipper(URL url) throws IOException, URISyntaxException {
-        super(new URL(url.toExternalForm().replace("thumbs.", "")));
+        super(new URI(url.toExternalForm().replace("thumbs.", "")).toURL());
     }
 
     @Override
@@ -47,12 +49,12 @@ public class RedgifsRipper extends AbstractHTMLRipper {
     }
 
     @Override
-    public URL sanitizeURL(URL url) throws MalformedURLException {
+    public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
         String sUrl = url.toExternalForm();
         sUrl = sUrl.replace("/gifs/detail", "");
         sUrl = sUrl.replace("/amp", "");
         sUrl = sUrl.replace("gifdeliverynetwork.com", "redgifs.com/watch");
-        return new URL(sUrl);
+        return new URI(sUrl).toURL();
     }
 
     public Matcher isProfile() {
@@ -72,16 +74,20 @@ public class RedgifsRipper extends AbstractHTMLRipper {
 
     @Override
     public Document getFirstPage() throws IOException {
-        if (!isProfile().matches() && !isSearch().matches()) {
+        try {
-            return Http.url(url).get();
+            if (!isProfile().matches() && !isSearch().matches()) {
-        } else if (isSearch().matches()) {
+                return Http.url(url).get();
-            searchText = getGID(url).replace("-", " ");
+            } else if (isSearch().matches()) {
-            return Http.url(
+                searchText = getGID(url).replace("-", " ");
-                    new URL("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart*searchCount)).ignoreContentType().get();
+                return Http.url(
-        } else {
+                        new URI("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart * searchCount).toURL()).ignoreContentType().get();
-            username = getGID(url);
+            } else {
-            return Http.url(new URL("https://api.redgifs.com/v1/users/" +  username + "/gfycats?count=" + count))
+                username = getGID(url);
-                       .ignoreContentType().get();
+                return Http.url(new URI("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count).toURL())
+                        .ignoreContentType().get();
+            }
+        } catch (URISyntaxException e) {
+            throw new IOException(e);
         }
     }
 
@@ -124,18 +130,18 @@ public class RedgifsRipper extends AbstractHTMLRipper {
     }
 
     @Override
-    public Document getNextPage(Document doc) throws IOException {
+    public Document getNextPage(Document doc) throws IOException, URISyntaxException {
         if (isSearch().matches()) {
             Document d = Http.url(
-                    new URL("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText
+                    new URI("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText
-                                    + "&count=" + searchCount + "&start=" + searchCount*++searchStart))
+                                    + "&count=" + searchCount + "&start=" + searchCount*++searchStart).toURL())
                        .ignoreContentType().get();
             return (hasURLs(d).isEmpty()) ? null : d;
         } else {
             if (cursor.equals("") || cursor.equals("null")) {
                 return null;
             } else {
-                Document d =  Http.url(new URL("https://api.redgifs.com/v1/users/" +  username + "/gfycats?count=" + count + "&cursor=" + cursor)).ignoreContentType().get();
+                Document d =  Http.url(new URI("https://api.redgifs.com/v1/users/" +  username + "/gfycats?count=" + count + "&cursor=" + cursor).toURL()).ignoreContentType().get();
                 return (hasURLs(d).isEmpty()) ? null : d;
             }
         }
@@ -182,11 +188,11 @@ public class RedgifsRipper extends AbstractHTMLRipper {
      * @return URL to video
      * @throws IOException
      */
-    public static String getVideoURL(URL url) throws IOException {
+    public static String getVideoURL(URL url) throws IOException, URISyntaxException {
         LOGGER.info("Retrieving " + url.toExternalForm());
 
         //Sanitize the URL first
-        url = new URL(url.toExternalForm().replace("/gifs/detail", ""));
+        url = new URI(url.toExternalForm().replace("/gifs/detail", "")).toURL();
 
         Document doc = Http.url(url).get();
         Elements videos = doc.select("script");

src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers;
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
@@ -82,12 +84,12 @@ public class TapasticRipper extends AbstractHTMLRipper {
                 prefix.append(String.format("-%0" + imgLog + "dof%0" + imgLog + "d-", i + 1, images.size()));
                 prefix.append(episode.filename.replace(" ", "-"));
                 prefix.append("-");
-                addURLToDownload(new URL(link), prefix.toString());
+                addURLToDownload(new URI(link).toURL(), prefix.toString());
                 if (isThisATest()) {
                     break;
                 }
             }
-        } catch (IOException e) {
+        } catch (IOException | URISyntaxException e) {
             LOGGER.error("[!] Exception while downloading " + url, e);
         }
 

src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java

@@ -1,9 +1,11 @@
 package com.rarchives.ripme.ripper.rippers;
 
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -110,11 +112,11 @@ public class TsuminoRipper extends AbstractHTMLRipper {
     }
 
     @Override
-    public List<String> getURLsFromPage(Document doc) {
+    public List<String> getURLsFromPage(Document doc) throws UnsupportedEncodingException {
         JSONArray imageIds = getPageUrls();
         List<String> result = new ArrayList<>();
         for (int i = 0; i < imageIds.length(); i++) {
-            result.add("http://www.tsumino.com/Image/Object?name=" + URLEncoder.encode(imageIds.getString(i)));
+            result.add("http://www.tsumino.com/Image/Object?name=" + URLEncoder.encode(imageIds.getString(i), StandardCharsets.UTF_8.name()));
         }
 
         return result;

src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers;
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
@@ -70,11 +72,11 @@ public class VidbleRipper extends AbstractHTMLRipper {
         addURLToDownload(url, getPrefix(index));
     }
 
-    public static List<URL> getURLsFromPage(URL url) throws IOException {
+    public static List<URL> getURLsFromPage(URL url) throws IOException, URISyntaxException {
         List<URL> urls = new ArrayList<>();
         Document doc = Http.url(url).get();
         for (String stringURL : getURLsFromPageStatic(doc)) {
-            urls.add(new URL(stringURL));
+            urls.add(new URI(stringURL).toURL());
         }
         return urls;
     }

src/main/java/com/rarchives/ripme/utils/RipUtils.java

@@ -84,7 +84,7 @@ public class RipUtils {
             try {
                 logger.info("Getting vidble album " + url);
                 result.addAll(VidbleRipper.getURLsFromPage(url));
-            } catch (IOException e) {
+            } catch (IOException | URISyntaxException e) {
                 // Do nothing
                 logger.warn("Exception while retrieving vidble page:", e);
             }
@@ -94,7 +94,7 @@ public class RipUtils {
             try {
                 logger.info("Getting eroshare album " + url);
                 result.addAll(EroShareRipper.getURLs(url));
-            } catch (IOException e) {
+            } catch (IOException | URISyntaxException e) {
                 // Do nothing
                 logger.warn("Exception while retrieving eroshare page:", e);
             }