mirror/ripme
1
0
Fork 0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-09-01 01:51:56 +02:00
Code Issues Releases Wiki Activity

Merge pull request #2053 from RipMeApp/fix-deprecated-url-ctors

Browse Source 
Fix usage of deprecated URL constructors, which found a couple of issues
This commit is contained in:
soloturn
2025-01-02 11:53:39 +01:00
committed by GitHub
parent 3b9352fb14 692430cfcb
commit 24c441e993
8 changed files with 52 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -121,6 +121,7 @@ the following combinations of tags:
- testSlow runs tests with tag "slow". - testSlow runs tests with tag "slow".
- tests can be run by test class, or single test. Use "testAll" so it does - tests can be run by test class, or single test. Use "testAll" so it does
not matter if a test is tagged or not. not matter if a test is tagged or not.
- tests can give the full stack of an assertion, exception, or error if you pass `--info` to the command
```bash ```bash
./gradlew test ./gradlew test
@@ -129,6 +130,7 @@ the following combinations of tags:
./gradlew testSlow ./gradlew testSlow
./gradlew testAll --tests XhamsterRipperTest ./gradlew testAll --tests XhamsterRipperTest
./gradlew testAll --tests XhamsterRipperTest.testXhamster2Album ./gradlew testAll --tests XhamsterRipperTest.testXhamster2Album
./gradlew testAll --tests ChanRipperTest --info
``` ```
Please note that some tests may fail as sites change and our rippers Please note that some tests may fail as sites change and our rippers

23
src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java
View File

@@ -60,11 +60,15 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
public Document getNextPage(Document doc) throws IOException, URISyntaxException { public Document getNextPage(Document doc) throws IOException, URISyntaxException {
return null; return null;
} }
protected abstract List<String> getURLsFromPage(Document page) throws UnsupportedEncodingException;
protected abstract List<String> getURLsFromPage(Document page) throws UnsupportedEncodingException, URISyntaxException;
protected List<String> getDescriptionsFromPage(Document doc) throws IOException { protected List<String> getDescriptionsFromPage(Document doc) throws IOException {
throw new IOException("getDescriptionsFromPage not implemented"); // Do I do this or make an abstract function? throw new IOException("getDescriptionsFromPage not implemented"); // Do I do this or make an abstract function?
} }
protected abstract void downloadURL(URL url, int index); protected abstract void downloadURL(URL url, int index);
protected DownloadThreadPool getThreadPool() { protected DownloadThreadPool getThreadPool() {
return null; return null;
} }
@@ -167,7 +171,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
for (String imageURL : imageURLs) { for (String imageURL : imageURLs) {
index += 1; index += 1;
LOGGER.debug("Found image url #" + index + ": '" + imageURL + "'"); LOGGER.debug("Found image url #" + index + ": '" + imageURL + "'");
downloadURL(new URL(imageURL), index); downloadURL(new URI(imageURL).toURL(), index);
if (isStopped() || isThisATest()) { if (isStopped() || isThisATest()) {
break; break;
} }
@@ -182,19 +186,26 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
if (isStopped() || isThisATest()) { if (isStopped() || isThisATest()) {
break; break;
} }
textindex += 1; textindex += 1;
LOGGER.debug("Getting description from " + textURL); LOGGER.debug("Getting description from " + textURL);
String[] tempDesc = getDescription(textURL,doc); String[] tempDesc = getDescription(textURL,doc);
if (tempDesc != null) { if (tempDesc != null) {
if (Utils.getConfigBoolean("file.overwrite", false) || !(new File( URL url = new URI(textURL).toURL();
String filename = fileNameFromURL(url);
boolean fileExists = new File(
workingDir.getCanonicalPath() workingDir.getCanonicalPath()
+ "" + ""
+ File.separator + File.separator
+ getPrefix(index) + getPrefix(index)
+ (tempDesc.length > 1 ? tempDesc[1] : fileNameFromURL(new URL(textURL))) + (tempDesc.length > 1 ? tempDesc[1] : filename)
+ ".txt").exists())) { + ".txt").exists();
if (Utils.getConfigBoolean("file.overwrite", false) || !fileExists) {
LOGGER.debug("Got description from " + textURL); LOGGER.debug("Got description from " + textURL);
saveText(new URL(textURL), "", tempDesc[0], textindex, (tempDesc.length > 1 ? tempDesc[1] : fileNameFromURL(new URL(textURL)))); saveText(url, "", tempDesc[0], textindex, (tempDesc.length > 1 ? tempDesc[1] : filename));
sleep(descSleepTime()); sleep(descSleepTime());
} else { } else {
LOGGER.debug("Description from " + textURL + " already exists."); LOGGER.debug("Description from " + textURL + " already exists.");

3
src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java
View File

@@ -8,6 +8,7 @@ import org.json.JSONObject;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
@@ -94,7 +95,7 @@ public abstract class AbstractJSONRipper extends AbstractRipper {
index += 1; index += 1;
LOGGER.debug("Found image url #" + index+ ": " + imageURL); LOGGER.debug("Found image url #" + index+ ": " + imageURL);
downloadURL(new URL(imageURL), index); downloadURL(new URI(imageURL).toURL(), index);
} }
if (isStopped() || isThisATest()) { if (isStopped() || isThisATest()) {

7
src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
View File

@@ -6,6 +6,7 @@ import com.rarchives.ripme.utils.Utils;
import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.RipUtils;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
@@ -208,7 +209,7 @@ public class ChanRipper extends AbstractHTMLRipper {
return false; return false;
} }
@Override @Override
public List<String> getURLsFromPage(Document page) { public List<String> getURLsFromPage(Document page) throws URISyntaxException {
List<String> imageURLs = new ArrayList<>(); List<String> imageURLs = new ArrayList<>();
Pattern p; Matcher m; Pattern p; Matcher m;
for (Element link : page.select("a")) { for (Element link : page.select("a")) {
@@ -254,8 +255,8 @@ public class ChanRipper extends AbstractHTMLRipper {
//Copied code from RedditRipper, getFilesFromURL should also implement stuff like flickr albums //Copied code from RedditRipper, getFilesFromURL should also implement stuff like flickr albums
URL originalURL; URL originalURL;
try { try {
originalURL = new URL(href); originalURL = new URI(href).toURL();
} catch (MalformedURLException e) { } catch (MalformedURLException | URISyntaxException | IllegalArgumentException e) {
continue; continue;
} }

6
src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java
View File

@@ -10,6 +10,8 @@ import org.jsoup.nodes.Document;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.ArrayList; import java.util.ArrayList;
@@ -27,10 +29,10 @@ public class LusciousRipper extends AbstractHTMLRipper {
} }
@Override @Override
public URL sanitizeURL(URL url) throws MalformedURLException { public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException{
String URLToReturn = url.toExternalForm(); String URLToReturn = url.toExternalForm();
URLToReturn = URLToReturn.replaceAll("https?://(?:www\\.)?luscious\\.", "https://old.luscious."); URLToReturn = URLToReturn.replaceAll("https?://(?:www\\.)?luscious\\.", "https://old.luscious.");
URL san_url = new URL(URLToReturn); URL san_url = new URI(URLToReturn).toURL();
LOGGER.info("sanitized URL is " + san_url.toExternalForm()); LOGGER.info("sanitized URL is " + san_url.toExternalForm());
return san_url; return san_url;
} }

1
src/main/java/com/rarchives/ripme/ripper/rippers/NudeGalsRipper.java
View File

@@ -56,6 +56,7 @@ public class NudeGalsRipper extends AbstractHTMLRipper {
for (Element thumb : thumbs) { for (Element thumb : thumbs) {
String link = thumb.attr("src").replaceAll("thumbs/th_", ""); String link = thumb.attr("src").replaceAll("thumbs/th_", "");
String imgSrc = "http://nude-gals.com/" + link; String imgSrc = "http://nude-gals.com/" + link;
imgSrc = imgSrc.replaceAll(" ", "%20");
imageURLs.add(imgSrc); imageURLs.add(imgSrc);
} }

2
src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java
View File

@@ -82,7 +82,7 @@ public class XvideosRipper extends AbstractSingleFileRipper {
String[] lines = e.html().split("\n"); String[] lines = e.html().split("\n");
for (String line : lines) { for (String line : lines) {
if (line.contains("html5player.setVideoUrlHigh")) { if (line.contains("html5player.setVideoUrlHigh")) {
String videoURL = line.replaceAll("\t", "").replaceAll("html5player.setVideoUrlHigh\\(", "").replaceAll("\'", "").replaceAll("\\);", ""); String videoURL = line.strip().replaceAll("\t", "").replaceAll("html5player.setVideoUrlHigh\\(", "").replaceAll("\'", "").replaceAll("\\);", "");
results.add(videoURL); results.add(videoURL);
} }
} }

6
src/main/java/com/rarchives/ripme/ripper/rippers/video/MotherlessVideoRipper.java
View File

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.util.List; import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
@@ -51,7 +53,7 @@ public class MotherlessVideoRipper extends VideoRipper {
} }
@Override @Override
public void rip() throws IOException { public void rip() throws IOException, URISyntaxException {
LOGGER.info(" Retrieving " + this.url); LOGGER.info(" Retrieving " + this.url);
String html = Http.url(this.url).get().toString(); String html = Http.url(this.url).get().toString();
if (html.contains("__fileurl = '")) { if (html.contains("__fileurl = '")) {
@@ -62,7 +64,7 @@ public class MotherlessVideoRipper extends VideoRipper {
throw new IOException("Could not find video URL at " + url); throw new IOException("Could not find video URL at " + url);
} }
String vidUrl = vidUrls.get(0); String vidUrl = vidUrls.get(0);
addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url)); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url));
waitForThreads(); waitForThreads();
} }
} }