mirror/ripme
1
0
Fork 0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-09-02 10:23:47 +02:00
Code Issues Releases Wiki Activity

new URL deprecated, use URI

Browse Source
This commit is contained in:
soloturn
2023-10-27 21:59:28 +02:00
parent 74fe3a7356
commit dd6ac9bc12
14 changed files with 76 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java
View File

@@ -3,6 +3,7 @@ package com.rarchives.ripme.ripper;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
@@ -54,7 +55,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
public Document getNextPage(Document doc) throws IOException, URISyntaxException {
return null;
}
protected abstract List<String> getURLsFromPage(Document page);
protected abstract List<String> getURLsFromPage(Document page) throws UnsupportedEncodingException;
protected List<String> getDescriptionsFromPage(Document doc) throws IOException {
throw new IOException("getDescriptionsFromPage not implemented"); // Do I do this or make an abstract function?
}

9
src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
View File

@@ -1,10 +1,7 @@
package com.rarchives.ripme.ripper;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.SocketTimeoutException;
import java.net.URL;
import java.net.URLConnection;
import java.net.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Arrays;
@@ -152,7 +149,7 @@ class DownloadFileThread implements Runnable {
redirected = true;
}
String location = huc.getHeaderField("Location");
urlToDownload = new URL(location);
urlToDownload = new URI(location).toURL();
// Throw exception so download can be retried
throw new IOException("Redirect status code " + statusCode + " - redirect to " + location);
}
@@ -284,7 +281,7 @@ class DownloadFileThread implements Runnable {
"HTTP status code " + hse.getStatusCode() + " while downloading " + url.toExternalForm());
return;
}
} catch (IOException e) {
} catch (IOException | URISyntaxException e) {
logger.debug("IOException", e);
logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - "
+ e.getMessage());

6
src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java
View File

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
@@ -114,7 +116,7 @@ public class EightmusesRipper extends AbstractHTMLRipper {
try {
for (int i = 0; i != json.getJSONArray("pictures").length(); i++) {
image = "https://www.8muses.com/image/fl/" + json.getJSONArray("pictures").getJSONObject(i).getString("publicUri");
URL imageUrl = new URL(image);
URL imageUrl = new URI(image).toURL();
addURLToDownload(imageUrl, getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, getPrefixShort(x), "", null, true);
// X is our page index
x++;
@@ -123,7 +125,7 @@ public class EightmusesRipper extends AbstractHTMLRipper {
}
}
return imageURLs;
} catch (MalformedURLException e) {
} catch (MalformedURLException | URISyntaxException e) {
LOGGER.error("\"" + image + "\" is malformed");
}
}

8
src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java
View File

@@ -7,6 +7,8 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@@ -194,7 +196,7 @@ public class EroShareRipper extends AbstractHTMLRipper {
throw new MalformedURLException("eroshare album not found in " + url + ", expected https://eroshare.com/album or eroshae.com/album");
}
public static List<URL> getURLs(URL url) throws IOException{
public static List<URL> getURLs(URL url) throws IOException, URISyntaxException {
Response resp = Http.url(url)
.ignoreContentType()
@@ -208,7 +210,7 @@ public class EroShareRipper extends AbstractHTMLRipper {
for (Element img : imgs) {
if (img.hasClass("album-image")) {
String imageURL = img.attr("src");
URLs.add(new URL(imageURL));
URLs.add(new URI(imageURL).toURL());
}
}
//Videos
@@ -217,7 +219,7 @@ public class EroShareRipper extends AbstractHTMLRipper {
if (vid.hasClass("album-video")) {
Elements source = vid.getElementsByTag("source");
String videoURL = source.first().attr("src");
URLs.add(new URL(videoURL));
URLs.add(new URI(videoURL).toURL());
}
}

6
src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java
View File

@@ -9,6 +9,8 @@ import org.jsoup.select.Elements;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
@@ -89,8 +91,8 @@ public class ErofusRipper extends AbstractHTMLRipper {
Map<String,String> opts = new HashMap<String, String>();
opts.put("subdirectory", page.title().replaceAll(" \\| Erofus - Sex and Porn Comics", "").replaceAll(" ", "_"));
opts.put("prefix", getPrefix(x));
addURLToDownload(new URL(image), opts);
} catch (MalformedURLException e) {
addURLToDownload(new URI(image).toURL(), opts);
} catch (MalformedURLException | URISyntaxException e) {
LOGGER.info(e.getMessage());
}
x++;

14
src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java
View File

@@ -4,6 +4,8 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.text.MessageFormat;
import java.util.ArrayList;
@@ -51,11 +53,11 @@ public class ImagefapRipper extends AbstractHTMLRipper {
* Reformat given URL into the desired format (all images on single page)
*/
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
String gid = getGID(url);
String newURL = "https://www.imagefap.com/pictures/" + gid + "/random-string";
LOGGER.debug("Changed URL from " + url + " to " + newURL);
return new URL(newURL);
return new URI(newURL).toURL();
}
@Override
@@ -106,7 +108,7 @@ public class ImagefapRipper extends AbstractHTMLRipper {
}
@Override
public Document getNextPage(Document doc) throws IOException {
public Document getNextPage(Document doc) throws IOException, URISyntaxException {
String nextURL = null;
for (Element a : doc.select("a.link3")) {
if (a.text().contains("next")) {
@@ -124,7 +126,7 @@ public class ImagefapRipper extends AbstractHTMLRipper {
LOGGER.info("Attempting to load next page URL: " + nextURL);
// Load next page
Document nextPage = getPageWithRetries(new URL(nextURL));
Document nextPage = getPageWithRetries(new URI(nextURL).toURL());
return nextPage;
}
@@ -182,7 +184,7 @@ public class ImagefapRipper extends AbstractHTMLRipper {
// Sleep before fetching image.
sleep(IMAGE_SLEEP_TIME);
Document doc = getPageWithRetries(new URL(pageURL));
Document doc = getPageWithRetries(new URI(pageURL).toURL());
String framedPhotoUrl = doc.select("img#mainPhoto").attr("data-src");
@@ -204,7 +206,7 @@ public class ImagefapRipper extends AbstractHTMLRipper {
return fullSizedUrl;
} catch (IOException e) {
} catch (IOException | URISyntaxException e) {
LOGGER.debug("Unable to get full size image URL from page: " + pageURL + " because: " + e.getMessage());
return null;
}

6
src/main/java/com/rarchives/ripme/ripper/rippers/JagodibujaRipper.java
View File

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@@ -56,8 +58,8 @@ public class JagodibujaRipper extends AbstractHTMLRipper {
Element elem = comicPage.select("span.full-size-link > a").first();
LOGGER.info("Got link " + elem.attr("href"));
try {
addURLToDownload(new URL(elem.attr("href")), "");
} catch (MalformedURLException e) {
addURLToDownload(new URI(elem.attr("href")).toURL(), "");
} catch (MalformedURLException | URISyntaxException e) {
LOGGER.warn("Malformed URL");
e.printStackTrace();
}

7
src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java
View File

@@ -9,6 +9,7 @@ import org.jsoup.select.Elements;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
@@ -95,7 +96,7 @@ public class LusciousRipper extends AbstractHTMLRipper {
sanitizedUrl = sanitizedUrl.replaceFirst(
"^https?://(?:members\\.|legacy\\.|www\\.)?luscious.net",
"https://legacy.luscious.net");
return new URL(sanitizedUrl);
return new URI(sanitizedUrl).toURL();
}
throw new Exception("ERROR: Unable to sanitize url.");
@@ -142,9 +143,9 @@ public class LusciousRipper extends AbstractHTMLRipper {
}
//If a valid download url was found.
addURLToDownload(new URL(downloadUrl), getPrefix(index));
addURLToDownload(new URI(downloadUrl).toURL(), getPrefix(index));
} catch (IOException e) {
} catch (IOException | URISyntaxException e) {
LOGGER.error("Error downloadiong url " + url, e);
}
}

3
src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java
View File

@@ -8,6 +8,7 @@ import org.jsoup.select.Elements;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
@@ -44,7 +45,7 @@ public class MultpornRipper extends AbstractHTMLRipper {
p = Pattern.compile("/node/(\\d+)/.*");
m = p.matcher(nodeHref);
if (m.matches()) {
this.url = new URL("https://multporn.net" + nodeHref);
this.url = new URI("https://multporn.net" + nodeHref).toURL();
return m.group(1);
}
}catch (Exception ignored){};

46
src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java
View File

@@ -11,6 +11,8 @@ import org.jsoup.select.Elements;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@@ -29,8 +31,8 @@ public class RedgifsRipper extends AbstractHTMLRipper {
int searchCount = 150;
int searchStart = 0;
public RedgifsRipper(URL url) throws IOException {
super(new URL(url.toExternalForm().replace("thumbs.", "")));
public RedgifsRipper(URL url) throws IOException, URISyntaxException {
super(new URI(url.toExternalForm().replace("thumbs.", "")).toURL());
}
@Override
@@ -47,12 +49,12 @@ public class RedgifsRipper extends AbstractHTMLRipper {
}
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
String sUrl = url.toExternalForm();
sUrl = sUrl.replace("/gifs/detail", "");
sUrl = sUrl.replace("/amp", "");
sUrl = sUrl.replace("gifdeliverynetwork.com", "redgifs.com/watch");
return new URL(sUrl);
return new URI(sUrl).toURL();
}
public Matcher isProfile() {
@@ -72,16 +74,20 @@ public class RedgifsRipper extends AbstractHTMLRipper {
@Override
public Document getFirstPage() throws IOException {
if (!isProfile().matches() && !isSearch().matches()) {
return Http.url(url).get();
} else if (isSearch().matches()) {
searchText = getGID(url).replace("-", " ");
return Http.url(
new URL("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart*searchCount)).ignoreContentType().get();
} else {
username = getGID(url);
return Http.url(new URL("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count))
.ignoreContentType().get();
try {
if (!isProfile().matches() && !isSearch().matches()) {
return Http.url(url).get();
} else if (isSearch().matches()) {
searchText = getGID(url).replace("-", " ");
return Http.url(
new URI("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart * searchCount).toURL()).ignoreContentType().get();
} else {
username = getGID(url);
return Http.url(new URI("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count).toURL())
.ignoreContentType().get();
}
} catch (URISyntaxException e) {
throw new IOException(e);
}
}
@@ -124,18 +130,18 @@ public class RedgifsRipper extends AbstractHTMLRipper {
}
@Override
public Document getNextPage(Document doc) throws IOException {
public Document getNextPage(Document doc) throws IOException, URISyntaxException {
if (isSearch().matches()) {
Document d = Http.url(
new URL("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText
+ "&count=" + searchCount + "&start=" + searchCount*++searchStart))
new URI("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText
+ "&count=" + searchCount + "&start=" + searchCount*++searchStart).toURL())
.ignoreContentType().get();
return (hasURLs(d).isEmpty()) ? null : d;
} else {
if (cursor.equals("") || cursor.equals("null")) {
return null;
} else {
Document d = Http.url(new URL("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor)).ignoreContentType().get();
Document d = Http.url(new URI("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor).toURL()).ignoreContentType().get();
return (hasURLs(d).isEmpty()) ? null : d;
}
}
@@ -182,11 +188,11 @@ public class RedgifsRipper extends AbstractHTMLRipper {
* @return URL to video
* @throws IOException
*/
public static String getVideoURL(URL url) throws IOException {
public static String getVideoURL(URL url) throws IOException, URISyntaxException {
LOGGER.info("Retrieving " + url.toExternalForm());
//Sanitize the URL first
url = new URL(url.toExternalForm().replace("/gifs/detail", ""));
url = new URI(url.toExternalForm().replace("/gifs/detail", "")).toURL();
Document doc = Http.url(url).get();
Elements videos = doc.select("script");

6
src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java
View File

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@@ -82,12 +84,12 @@ public class TapasticRipper extends AbstractHTMLRipper {
prefix.append(String.format("-%0" + imgLog + "dof%0" + imgLog + "d-", i + 1, images.size()));
prefix.append(episode.filename.replace(" ", "-"));
prefix.append("-");
addURLToDownload(new URL(link), prefix.toString());
addURLToDownload(new URI(link).toURL(), prefix.toString());
if (isThisATest()) {
break;
}
}
} catch (IOException e) {
} catch (IOException | URISyntaxException e) {
LOGGER.error("[!] Exception while downloading " + url, e);
}

6
src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java
View File

@@ -1,9 +1,11 @@
package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -110,11 +112,11 @@ public class TsuminoRipper extends AbstractHTMLRipper {
}
@Override
public List<String> getURLsFromPage(Document doc) {
public List<String> getURLsFromPage(Document doc) throws UnsupportedEncodingException {
JSONArray imageIds = getPageUrls();
List<String> result = new ArrayList<>();
for (int i = 0; i < imageIds.length(); i++) {
result.add("http://www.tsumino.com/Image/Object?name=" + URLEncoder.encode(imageIds.getString(i)));
result.add("http://www.tsumino.com/Image/Object?name=" + URLEncoder.encode(imageIds.getString(i), StandardCharsets.UTF_8.name()));
}
return result;

6
src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java
View File

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@@ -70,11 +72,11 @@ public class VidbleRipper extends AbstractHTMLRipper {
addURLToDownload(url, getPrefix(index));
}
public static List<URL> getURLsFromPage(URL url) throws IOException {
public static List<URL> getURLsFromPage(URL url) throws IOException, URISyntaxException {
List<URL> urls = new ArrayList<>();
Document doc = Http.url(url).get();
for (String stringURL : getURLsFromPageStatic(doc)) {
urls.add(new URL(stringURL));
urls.add(new URI(stringURL).toURL());
}
return urls;
}

4
src/main/java/com/rarchives/ripme/utils/RipUtils.java
View File

@@ -84,7 +84,7 @@ public class RipUtils {
try {
logger.info("Getting vidble album " + url);
result.addAll(VidbleRipper.getURLsFromPage(url));
} catch (IOException e) {
} catch (IOException | URISyntaxException e) {
// Do nothing
logger.warn("Exception while retrieving vidble page:", e);
}
@@ -94,7 +94,7 @@ public class RipUtils {
try {
logger.info("Getting eroshare album " + url);
result.addAll(EroShareRipper.getURLs(url));
} catch (IOException e) {
} catch (IOException | URISyntaxException e) {
// Do nothing
logger.warn("Exception while retrieving eroshare page:", e);
}