mirror/ripme
1
0
Fork 0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-08-24 14:33:17 +02:00
Code Issues Releases Wiki Activity

sanitizeURL can throw URISyntaxException, imgur.

Browse Source
This commit is contained in:
soloturn
2023-06-15 14:54:26 +02:00
parent 5aeaf7d2b0
commit 07178479e9
3 changed files with 57 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java
View File

@@ -7,6 +7,7 @@ import com.rarchives.ripme.utils.Utils;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
@@ -33,7 +34,7 @@ public abstract class AlbumRipper extends AbstractRipper {
} }
public abstract boolean canRip(URL url); public abstract boolean canRip(URL url);
public abstract URL sanitizeURL(URL url) throws MalformedURLException; public abstract URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException;
public abstract void rip() throws IOException; public abstract void rip() throws IOException;
public abstract String getHost(); public abstract String getHost();
public abstract String getGID(URL url) throws MalformedURLException; public abstract String getGID(URL url) throws MalformedURLException;

44
src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java
View File

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
@@ -72,7 +74,7 @@ public class ImgurRipper extends AlbumRipper {
return true; return true;
} }
public URL sanitizeURL(URL url) throws MalformedURLException { public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
String u = url.toExternalForm(); String u = url.toExternalForm();
if (u.indexOf('#') >= 0) { if (u.indexOf('#') >= 0) {
u = u.substring(0, u.indexOf('#')); u = u.substring(0, u.indexOf('#'));
@@ -80,7 +82,7 @@ public class ImgurRipper extends AlbumRipper {
u = u.replace("imgur.com/gallery/", "imgur.com/a/"); u = u.replace("imgur.com/gallery/", "imgur.com/a/");
u = u.replace("https?://m\\.imgur\\.com", "http://imgur.com"); u = u.replace("https?://m\\.imgur\\.com", "http://imgur.com");
u = u.replace("https?://i\\.imgur\\.com", "http://imgur.com"); u = u.replace("https?://i\\.imgur\\.com", "http://imgur.com");
return new URL(u); return new URI(u).toURL();
} }
public String getAlbumTitle(URL url) throws MalformedURLException { public String getAlbumTitle(URL url) throws MalformedURLException {
@@ -145,6 +147,7 @@ public class ImgurRipper extends AlbumRipper {
@Override @Override
public void rip() throws IOException { public void rip() throws IOException {
try {
switch (albumType) { switch (albumType) {
case ALBUM: case ALBUM:
// Fall-through // Fall-through
@@ -175,10 +178,13 @@ public class ImgurRipper extends AlbumRipper {
ripUserImages(url); ripUserImages(url);
break; break;
} }
} catch (URISyntaxException e) {
throw new IOException("Failed ripping " + this.url, e);
}
waitForThreads(); waitForThreads();
} }
private void ripSingleImage(URL url) throws IOException { private void ripSingleImage(URL url) throws IOException, URISyntaxException {
String strUrl = url.toExternalForm(); String strUrl = url.toExternalForm();
Document document = getDocument(strUrl); Document document = getDocument(strUrl);
Matcher m = getEmbeddedJsonMatcher(document); Matcher m = getEmbeddedJsonMatcher(document);
@@ -188,11 +194,11 @@ public class ImgurRipper extends AlbumRipper {
} }
} }
private void ripAlbum(URL url) throws IOException { private void ripAlbum(URL url) throws IOException, URISyntaxException {
ripAlbum(url, ""); ripAlbum(url, "");
} }
private void ripAlbum(URL url, String subdirectory) throws IOException { private void ripAlbum(URL url, String subdirectory) throws IOException, URISyntaxException {
int index; int index;
this.sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm()); this.sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm());
index = 0; index = 0;
@@ -238,7 +244,7 @@ public class ImgurRipper extends AlbumRipper {
continue; continue;
} }
String original = links.getString("original"); String original = links.getString("original");
ImgurImage theImage = new ImgurImage(new URL(original)); ImgurImage theImage = new ImgurImage(new URI(original).toURL());
album.addImage(theImage); album.addImage(theImage);
} catch (Exception e) { } catch (Exception e) {
LOGGER.error("Got exception while fetching imgur ID " + imageId, e); LOGGER.error("Got exception while fetching imgur ID " + imageId, e);
@@ -248,7 +254,7 @@ public class ImgurRipper extends AlbumRipper {
return album; return album;
} }
public static ImgurAlbum getImgurAlbum(URL url) throws IOException { public static ImgurAlbum getImgurAlbum(URL url) throws IOException, URISyntaxException {
String strUrl = url.toExternalForm(); String strUrl = url.toExternalForm();
if (!strUrl.contains(",")) { if (!strUrl.contains(",")) {
strUrl += "/all"; strUrl += "/all";
@@ -262,7 +268,7 @@ public class ImgurRipper extends AlbumRipper {
JSONObject json = new JSONObject(Jsoup.clean(doc.body().toString(), Safelist.none())); JSONObject json = new JSONObject(Jsoup.clean(doc.body().toString(), Safelist.none()));
JSONArray jsonImages = json.getJSONObject("data").getJSONArray("images"); JSONArray jsonImages = json.getJSONObject("data").getJSONArray("images");
return createImgurAlbumFromJsonArray(url, jsonImages); return createImgurAlbumFromJsonArray(url, jsonImages);
} catch (JSONException e) { } catch (JSONException | URISyntaxException e) {
LOGGER.debug("Error while parsing JSON at " + url + ", continuing", e); LOGGER.debug("Error while parsing JSON at " + url + ", continuing", e);
} }
@@ -296,7 +302,7 @@ public class ImgurRipper extends AlbumRipper {
if (image.endsWith(".gif") && Utils.getConfigBoolean("prefer.mp4", false)) { if (image.endsWith(".gif") && Utils.getConfigBoolean("prefer.mp4", false)) {
image = image.replace(".gif", ".mp4"); image = image.replace(".gif", ".mp4");
} }
ImgurImage imgurImage = new ImgurImage(new URL(image)); ImgurImage imgurImage = new ImgurImage(new URI(image).toURL());
imgurAlbum.addImage(imgurImage); imgurAlbum.addImage(imgurImage);
} }
return imgurAlbum; return imgurAlbum;
@@ -307,25 +313,25 @@ public class ImgurRipper extends AlbumRipper {
return p.matcher(doc.body().html()); return p.matcher(doc.body().html());
} }
private static ImgurAlbum createImgurAlbumFromJsonArray(URL url, JSONArray jsonImages) throws MalformedURLException { private static ImgurAlbum createImgurAlbumFromJsonArray(URL url, JSONArray jsonImages) throws MalformedURLException, URISyntaxException {
ImgurAlbum imgurAlbum = new ImgurAlbum(url); ImgurAlbum imgurAlbum = new ImgurAlbum(url);
int imagesLength = jsonImages.length(); int imagesLength = jsonImages.length();
for (int i = 0; i < imagesLength; i++) { for (int i = 0; i < imagesLength; i++) {
JSONObject ob = jsonImages.getJSONObject(i); JSONObject ob = jsonImages.getJSONObject(i);
imgurAlbum.addImage(new ImgurImage( new URL(ob.getString("link")))); imgurAlbum.addImage(new ImgurImage( new URI(ob.getString("link")).toURL()));
} }
return imgurAlbum; return imgurAlbum;
} }
private static URL extractImageUrlFromJson(JSONObject json) throws MalformedURLException { private static URL extractImageUrlFromJson(JSONObject json) throws MalformedURLException, URISyntaxException {
String ext = json.getString("ext"); String ext = json.getString("ext");
if (ext.equals(".gif") && Utils.getConfigBoolean("prefer.mp4", false)) { if (ext.equals(".gif") && Utils.getConfigBoolean("prefer.mp4", false)) {
ext = ".mp4"; ext = ".mp4";
} }
return new URL( return new URI(
"http://i.imgur.com/" "http://i.imgur.com/"
+ json.getString("hash") + json.getString("hash")
+ ext); + ext).toURL();
} }
private static Document getDocument(String strUrl) throws IOException { private static Document getDocument(String strUrl) throws IOException {
@@ -352,7 +358,7 @@ public class ImgurRipper extends AlbumRipper {
* @param url * @param url
* URL to imgur user account (http://username.imgur.com) * URL to imgur user account (http://username.imgur.com)
*/ */
private void ripUserAccount(URL url) throws IOException { private void ripUserAccount(URL url) throws IOException, URISyntaxException {
LOGGER.info("Retrieving " + url); LOGGER.info("Retrieving " + url);
sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm()); sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm());
Document doc = Http.url(url).get(); Document doc = Http.url(url).get();
@@ -363,7 +369,7 @@ public class ImgurRipper extends AlbumRipper {
continue; continue;
} }
String albumID = album.attr("href").substring(album.attr("href").lastIndexOf('/') + 1); String albumID = album.attr("href").substring(album.attr("href").lastIndexOf('/') + 1);
URL albumURL = new URL("http:" + album.attr("href") + "/noscript"); URL albumURL = new URI("http:" + album.attr("href") + "/noscript").toURL();
try { try {
ripAlbum(albumURL, albumID); ripAlbum(albumURL, albumID);
Thread.sleep(SLEEP_BETWEEN_ALBUMS * 1000L); Thread.sleep(SLEEP_BETWEEN_ALBUMS * 1000L);
@@ -398,7 +404,7 @@ public class ImgurRipper extends AlbumRipper {
if (Utils.getConfigBoolean("download.save_order", true)) { if (Utils.getConfigBoolean("download.save_order", true)) {
prefix = String.format("%03d_", imagesFound); prefix = String.format("%03d_", imagesFound);
} }
addURLToDownload(new URL(imageUrl), prefix); addURLToDownload(new URI(imageUrl).toURL(), prefix);
} }
if (imagesFound >= imagesTotal) { if (imagesFound >= imagesTotal) {
break; break;
@@ -411,7 +417,7 @@ public class ImgurRipper extends AlbumRipper {
} }
} }
private void ripSubreddit(URL url) throws IOException { private void ripSubreddit(URL url) throws IOException, URISyntaxException {
int page = 0; int page = 0;
while (true) { while (true) {
stopCheck(); stopCheck();
@@ -431,7 +437,7 @@ public class ImgurRipper extends AlbumRipper {
if (image.contains("b.")) { if (image.contains("b.")) {
image = image.replace("b.", "."); image = image.replace("b.", ".");
} }
URL imageURL = new URL(image); URL imageURL = new URI(image).toURL();
addURLToDownload(imageURL); addURLToDownload(imageURL);
} }
if (imgs.isEmpty()) { if (imgs.isEmpty()) {

2
src/main/java/com/rarchives/ripme/utils/RipUtils.java
View File

@@ -42,7 +42,7 @@ public class RipUtils {
logger.debug("Got imgur image: " + imgurImage.url); logger.debug("Got imgur image: " + imgurImage.url);
result.add(imgurImage.url); result.add(imgurImage.url);
} }
} catch (IOException e) { } catch (IOException | URISyntaxException e) {
logger.error("[!] Exception while loading album " + url, e); logger.error("[!] Exception while loading album " + url, e);
} }
return result; return result;