1
0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-08-18 11:41:21 +02:00

new URI instead of new URL.

This commit is contained in:
soloturn
2024-03-20 08:54:52 +01:00
parent c2d1472008
commit bce4ddd74d
18 changed files with 88 additions and 92 deletions

View File

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@@ -50,7 +52,7 @@ public class ArtStationRipper extends AbstractJSONRipper {
try { try {
// groupData = Http.url(albumURL.getLocation()).getJSON(); // groupData = Http.url(albumURL.getLocation()).getJSON();
groupData = getJson(albumURL.getLocation()); groupData = getJson(albumURL.getLocation());
} catch (IOException e) { } catch (IOException | URISyntaxException e) {
throw new MalformedURLException("Couldn't load JSON from " + albumURL.getLocation()); throw new MalformedURLException("Couldn't load JSON from " + albumURL.getLocation());
} }
return groupData.getString("title"); return groupData.getString("title");
@@ -62,7 +64,7 @@ public class ArtStationRipper extends AbstractJSONRipper {
try { try {
// groupData = Http.url(userInfoURL).getJSON(); // groupData = Http.url(userInfoURL).getJSON();
groupData = getJson(userInfoURL); groupData = getJson(userInfoURL);
} catch (IOException e) { } catch (IOException | URISyntaxException e) {
throw new MalformedURLException("Couldn't load JSON from " + userInfoURL); throw new MalformedURLException("Couldn't load JSON from " + userInfoURL);
} }
return groupData.getString("full_name"); return groupData.getString("full_name");
@@ -74,7 +76,7 @@ public class ArtStationRipper extends AbstractJSONRipper {
} }
@Override @Override
protected JSONObject getFirstPage() throws IOException { protected JSONObject getFirstPage() throws IOException, URISyntaxException {
if (albumURL.getType() == URL_TYPE.SINGLE_PROJECT) { if (albumURL.getType() == URL_TYPE.SINGLE_PROJECT) {
// URL points to JSON of a single project, just return it // URL points to JSON of a single project, just return it
// return Http.url(albumURL.getLocation()).getJSON(); // return Http.url(albumURL.getLocation()).getJSON();
@@ -90,7 +92,7 @@ public class ArtStationRipper extends AbstractJSONRipper {
if (albumContent.getInt("total_count") > 0) { if (albumContent.getInt("total_count") > 0) {
// Get JSON of the first project and return it // Get JSON of the first project and return it
JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(0); JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(0);
ParsedURL projectURL = parseURL(new URL(projectInfo.getString("permalink"))); ParsedURL projectURL = parseURL(new URI(projectInfo.getString("permalink")).toURL());
// return Http.url(projectURL.getLocation()).getJSON(); // return Http.url(projectURL.getLocation()).getJSON();
return getJson(projectURL.getLocation()); return getJson(projectURL.getLocation());
} }
@@ -100,7 +102,7 @@ public class ArtStationRipper extends AbstractJSONRipper {
} }
@Override @Override
protected JSONObject getNextPage(JSONObject doc) throws IOException { protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException {
if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) { if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) {
// Initialize the page number if it hasn't been initialized already // Initialize the page number if it hasn't been initialized already
if (projectPageNumber == null) { if (projectPageNumber == null) {
@@ -117,7 +119,7 @@ public class ArtStationRipper extends AbstractJSONRipper {
projectIndex = 0; projectIndex = 0;
} }
Integer currentProject = ((projectPageNumber - 1) * 50) + (projectIndex + 1); int currentProject = ((projectPageNumber - 1) * 50) + (projectIndex + 1);
// JSONObject albumContent = Http.url(albumURL.getLocation() + "?page=" + // JSONObject albumContent = Http.url(albumURL.getLocation() + "?page=" +
// projectPageNumber).getJSON(); // projectPageNumber).getJSON();
JSONObject albumContent = getJson(albumURL.getLocation() + "?page=" + projectPageNumber); JSONObject albumContent = getJson(albumURL.getLocation() + "?page=" + projectPageNumber);
@@ -125,7 +127,7 @@ public class ArtStationRipper extends AbstractJSONRipper {
if (albumContent.getInt("total_count") > currentProject) { if (albumContent.getInt("total_count") > currentProject) {
// Get JSON of the next project and return it // Get JSON of the next project and return it
JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(projectIndex); JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(projectIndex);
ParsedURL projectURL = parseURL(new URL(projectInfo.getString("permalink"))); ParsedURL projectURL = parseURL(new URI(projectInfo.getString("permalink")).toURL());
projectIndex++; projectIndex++;
// return Http.url(projectURL.getLocation()).getJSON(); // return Http.url(projectURL.getLocation()).getJSON();
return getJson(projectURL.getLocation()); return getJson(projectURL.getLocation());
@@ -320,8 +322,8 @@ public class ArtStationRipper extends AbstractJSONRipper {
throw new IOException("Error fetching json. Status code:" + status); throw new IOException("Error fetching json. Status code:" + status);
} }
private JSONObject getJson(String url) throws IOException { private JSONObject getJson(String url) throws IOException, URISyntaxException {
return getJson(new URL(url)); return getJson(new URI(url).toURL());
} }
} }

View File

@@ -72,7 +72,7 @@ public class ChanRipper extends AbstractHTMLRipper {
); );
private ChanSite chanSite; private ChanSite chanSite;
private Boolean generalChanSite = true; private boolean generalChanSite = true;
public ChanRipper(URL url) throws IOException { public ChanRipper(URL url) throws IOException {
super(url); super(url);

View File

@@ -94,13 +94,6 @@ public class EHentaiRipper extends AbstractHTMLRipper {
+ " Got: " + url); + " Got: " + url);
} }
/**
* Attempts to get page, checks for IP ban, waits.
*
* @param url
* @return Page document
* @throws IOException If page loading errors, or if retries are exhausted
*/
private Document getPageWithRetries(URL url) throws IOException { private Document getPageWithRetries(URL url) throws IOException {
Document doc; Document doc;
int retries = 3; int retries = 3;
@@ -251,16 +244,16 @@ public class EHentaiRipper extends AbstractHTMLRipper {
savePath += String.format("%03d_", index); savePath += String.format("%03d_", index);
} }
savePath += m.group(1); savePath += m.group(1);
addURLToDownload(new URL(imgsrc), Paths.get(savePath)); addURLToDownload(new URI(imgsrc).toURL(), Paths.get(savePath));
} else { } else {
// Provide prefix and let the AbstractRipper "guess" the filename // Provide prefix and let the AbstractRipper "guess" the filename
String prefix = ""; String prefix = "";
if (Utils.getConfigBoolean("download.save_order", true)) { if (Utils.getConfigBoolean("download.save_order", true)) {
prefix = String.format("%03d_", index); prefix = String.format("%03d_", index);
} }
addURLToDownload(new URL(imgsrc), prefix); addURLToDownload(new URI(imgsrc).toURL(), prefix);
} }
} catch (IOException e) { } catch (IOException | URISyntaxException e) {
LOGGER.error("[!] Exception while loading/parsing " + this.url, e); LOGGER.error("[!] Exception while loading/parsing " + this.url, e);
} }
} }

View File

@@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
@@ -85,8 +86,8 @@ public class EromeRipper extends AbstractHTMLRipper {
} }
@Override @Override
public URL sanitizeURL(URL url) throws MalformedURLException { public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
return new URL(url.toExternalForm().replaceAll("https?://erome.com", "https://www.erome.com")); return new URI(url.toExternalForm().replaceAll("https?://erome.com", "https://www.erome.com")).toURL();
} }

View File

@@ -1,9 +1,7 @@
package com.rarchives.ripme.ripper.rippers; package com.rarchives.ripme.ripper.rippers;
import java.io.IOException; import java.io.IOException;
import java.net.HttpURLConnection; import java.net.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
@@ -164,8 +162,8 @@ public class FivehundredpxRipper extends AbstractJSONRipper {
} }
@Override @Override
public JSONObject getFirstPage() throws IOException { public JSONObject getFirstPage() throws IOException, URISyntaxException {
URL apiURL = new URL(baseURL + "&consumer_key=" + CONSUMER_KEY); URL apiURL = new URI(baseURL + "&consumer_key=" + CONSUMER_KEY).toURL();
LOGGER.debug("apiURL: " + apiURL); LOGGER.debug("apiURL: " + apiURL);
JSONObject json = Http.url(apiURL).getJSON(); JSONObject json = Http.url(apiURL).getJSON();
@@ -232,7 +230,7 @@ public class FivehundredpxRipper extends AbstractJSONRipper {
} }
@Override @Override
public JSONObject getNextPage(JSONObject json) throws IOException { public JSONObject getNextPage(JSONObject json) throws IOException, URISyntaxException {
if (isThisATest()) { if (isThisATest()) {
return null; return null;
} }
@@ -249,9 +247,9 @@ public class FivehundredpxRipper extends AbstractJSONRipper {
sleep(500); sleep(500);
++page; ++page;
URL apiURL = new URL(baseURL URL apiURL = new URI(baseURL
+ "&page=" + page + "&page=" + page
+ "&consumer_key=" + CONSUMER_KEY); + "&consumer_key=" + CONSUMER_KEY).toURL();
return Http.url(apiURL).getJSON(); return Http.url(apiURL).getJSON();
} }
@@ -296,14 +294,9 @@ public class FivehundredpxRipper extends AbstractJSONRipper {
} }
} }
} }
if (imageURL == null) { imageURLs.add(imageURL);
LOGGER.error("Failed to find image for photo " + photo.toString()); if (isThisATest()) {
} break;
else {
imageURLs.add(imageURL);
if (isThisATest()) {
break;
}
} }
} }
return imageURLs; return imageURLs;
@@ -311,13 +304,13 @@ public class FivehundredpxRipper extends AbstractJSONRipper {
private boolean urlExists(String url) { private boolean urlExists(String url) {
try { try {
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); HttpURLConnection connection = (HttpURLConnection) new URI(url).toURL().openConnection();
connection.setRequestMethod("HEAD"); connection.setRequestMethod("HEAD");
if (connection.getResponseCode() != 200) { if (connection.getResponseCode() != 200) {
throw new IOException("Couldn't find full-size image at " + url); throw new IOException("Couldn't find full-size image at " + url);
} }
return true; return true;
} catch (IOException e) { } catch (IOException | URISyntaxException e) {
return false; return false;
} }
} }

View File

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@@ -40,7 +42,7 @@ public class FuskatorRipper extends AbstractHTMLRipper {
} }
@Override @Override
public URL sanitizeURL(URL url) throws MalformedURLException { public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
String u = url.toExternalForm(); String u = url.toExternalForm();
if (u.contains("/thumbs/")) { if (u.contains("/thumbs/")) {
u = u.replace("/thumbs/", "/full/"); u = u.replace("/thumbs/", "/full/");
@@ -48,7 +50,7 @@ public class FuskatorRipper extends AbstractHTMLRipper {
if (u.contains("/expanded/")) { if (u.contains("/expanded/")) {
u = u.replaceAll("/expanded/", "/full/"); u = u.replaceAll("/expanded/", "/full/");
} }
return new URL(u); return new URI(u).toURL();
} }
@Override @Override

View File

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Base64; import java.util.Base64;
@@ -80,16 +82,15 @@ public class HentaiNexusRipper extends AbstractJSONRipper {
} }
@Override @Override
protected JSONObject getFirstPage() throws IOException { protected JSONObject getFirstPage() throws IOException, URISyntaxException {
String jsonEncodedString = getJsonEncodedStringFromPage(); String jsonEncodedString = getJsonEncodedStringFromPage();
String jsonDecodedString = decodeJsonString(jsonEncodedString); String jsonDecodedString = decodeJsonString(jsonEncodedString);
return new JSONObject(jsonDecodedString); return new JSONObject(jsonDecodedString);
} }
public String getJsonEncodedStringFromPage() throws MalformedURLException, IOException public String getJsonEncodedStringFromPage() throws MalformedURLException, IOException, URISyntaxException {
{
// Image data only appears on the /read/ page and not on the /view/ one. // Image data only appears on the /read/ page and not on the /view/ one.
URL readUrl = new URL(String.format("http://hentainexus.com/read/%s",getGID(url))); URL readUrl = new URI(String.format("http://hentainexus.com/read/%s",getGID(url))).toURL();
Document document = Http.url(readUrl).response().parse(); Document document = Http.url(readUrl).response().parse();
for (Element scripts : document.getElementsByTag("script")) { for (Element scripts : document.getElementsByTag("script")) {
@@ -143,7 +144,7 @@ public class HentaiNexusRipper extends AbstractJSONRipper {
} }
magicByte = (byte) (magicByte & 0x7); magicByte = (byte) (magicByte & 0x7);
ArrayList<Integer> newArray = new ArrayList(); ArrayList<Integer> newArray = new ArrayList<>();
for (int i = 0x0; i < 0x100; i++) { for (int i = 0x0; i < 0x100; i++) {
newArray.add(i); newArray.add(i);

View File

@@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
@@ -14,7 +15,6 @@ import org.jsoup.nodes.Document;
import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Http;
import org.jsoup.nodes.Element;
public class HitomiRipper extends AbstractHTMLRipper { public class HitomiRipper extends AbstractHTMLRipper {
@@ -47,9 +47,9 @@ public class HitomiRipper extends AbstractHTMLRipper {
} }
@Override @Override
public Document getFirstPage() throws IOException { public Document getFirstPage() throws IOException, URISyntaxException {
// if we go to /GALLERYID.js we get a nice json array of all images in the gallery // if we go to /GALLERYID.js we get a nice json array of all images in the gallery
return Http.url(new URL(url.toExternalForm().replaceAll("hitomi", "ltn.hitomi").replaceAll(".html", ".js"))).ignoreContentType().get(); return Http.url(new URI(url.toExternalForm().replaceAll("hitomi", "ltn.hitomi").replaceAll(".html", ".js")).toURL()).ignoreContentType().get();
} }

View File

@@ -8,6 +8,8 @@ import org.json.JSONObject;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@@ -34,7 +36,7 @@ public class NsfwXxxRipper extends AbstractJSONRipper {
@Override @Override
public URL sanitizeURL(URL url) throws MalformedURLException { public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
String u = url.toExternalForm(); String u = url.toExternalForm();
// https://nsfw.xxx/user/kelly-kat/foo -> https://nsfw.xxx/user/kelly-kat // https://nsfw.xxx/user/kelly-kat/foo -> https://nsfw.xxx/user/kelly-kat
// https://nsfw.xxx/user/kelly-kat -> https://nsfw.xxx/user/kelly-kat // https://nsfw.xxx/user/kelly-kat -> https://nsfw.xxx/user/kelly-kat
@@ -44,15 +46,15 @@ public class NsfwXxxRipper extends AbstractJSONRipper {
throw new MalformedURLException("Invalid URL: " + url); throw new MalformedURLException("Invalid URL: " + url);
} }
return new URL(u); return new URI(u).toURL();
} }
String getUser() throws MalformedURLException { String getUser() throws MalformedURLException {
return getGID(url); return getGID(url);
} }
URL getPage(int page) throws MalformedURLException { URL getPage(int page) throws MalformedURLException, URISyntaxException {
return new URL("https://nsfw.xxx/slide-page/" + page + "?nsfw%5B%5D=0&types%5B%5D=image&types%5B%5D=video&types%5B%5D=gallery&slider=1&jsload=1&user=" + getUser()); return new URI("https://nsfw.xxx/slide-page/" + page + "?nsfw%5B%5D=0&types%5B%5D=image&types%5B%5D=video&types%5B%5D=gallery&slider=1&jsload=1&user=" + getUser()).toURL();
} }
@@ -71,18 +73,18 @@ public class NsfwXxxRipper extends AbstractJSONRipper {
int currentPage = 1; int currentPage = 1;
@Override @Override
protected JSONObject getFirstPage() throws IOException { protected JSONObject getFirstPage() throws IOException, URISyntaxException {
return Http.url(getPage(1)).getJSON(); return Http.url(getPage(1)).getJSON();
} }
List<String> descriptions = new ArrayList<>(); List<String> descriptions = new ArrayList<>();
@Override @Override
protected JSONObject getNextPage(JSONObject doc) throws IOException { protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException {
currentPage++; currentPage++;
JSONObject nextPage = Http.url(getPage(doc.getInt("page") + 1)).getJSON(); JSONObject nextPage = Http.url(getPage(doc.getInt("page") + 1)).getJSON();
JSONArray items = nextPage.getJSONArray("items"); JSONArray items = nextPage.getJSONArray("items");
if (items.length() == 0) { if (items.isEmpty()) {
throw new IOException("No more pages"); throw new IOException("No more pages");
} }
return nextPage; return nextPage;
@@ -120,7 +122,7 @@ public class NsfwXxxRipper extends AbstractJSONRipper {
return new ApiEntry(srcUrl, o.getString("author"), o.getString("title")); return new ApiEntry(srcUrl, o.getString("author"), o.getString("title"));
}) })
.collect(Collectors.toList()); .toList();
data.forEach(e -> descriptions.add(e.title)); data.forEach(e -> descriptions.add(e.title));
return data.stream().map(e -> e.srcUrl).collect(Collectors.toList()); return data.stream().map(e -> e.srcUrl).collect(Collectors.toList());

View File

@@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
@@ -44,13 +45,13 @@ public class XhamsterRipper extends AbstractHTMLRipper {
} }
@Override @Override
public URL sanitizeURL(URL url) throws MalformedURLException { public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
if (isVideoUrl(url)) { if (isVideoUrl(url)) {
return url; return url;
} }
String URLToReturn = url.toExternalForm(); String URLToReturn = url.toExternalForm();
URLToReturn = URLToReturn.replaceAll("https?://\\w?\\w?\\.?xhamster([^<]*)\\.", "https://m.xhamster$1."); URLToReturn = URLToReturn.replaceAll("https?://\\w?\\w?\\.?xhamster([^<]*)\\.", "https://m.xhamster$1.");
URL san_url = new URL(URLToReturn); URL san_url = new URI(URLToReturn).toURL();
LOGGER.info("sanitized URL is " + san_url.toExternalForm()); LOGGER.info("sanitized URL is " + san_url.toExternalForm());
return san_url; return san_url;
} }
@@ -168,10 +169,10 @@ public class XhamsterRipper extends AbstractHTMLRipper {
// This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to // This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to
// the page chamster.com but displays the mobile site from m.xhamster.com // the page chamster.com but displays the mobile site from m.xhamster.com
pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster([^<]*)\\.", "://m.xhamster$1."); pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster([^<]*)\\.", "://m.xhamster$1.");
String image = Http.url(new URL(pageWithImageUrl)).get().select("a > img#photoCurr").attr("src"); String image = Http.url(new URI(pageWithImageUrl).toURL()).get().select("a > img#photoCurr").attr("src");
result.add(image); result.add(image);
downloadFile(image); downloadFile(image);
} catch (IOException e) { } catch (IOException | URISyntaxException e) {
LOGGER.error("Was unable to load page " + pageWithImageUrl); LOGGER.error("Was unable to load page " + pageWithImageUrl);
} }
if (isStopped() || isThisATest()) { if (isStopped() || isThisATest()) {

View File

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@@ -54,7 +56,7 @@ public class CliphunterRipper extends VideoRipper {
} }
@Override @Override
public void rip() throws IOException { public void rip() throws IOException, URISyntaxException {
LOGGER.info("Retrieving " + this.url); LOGGER.info("Retrieving " + this.url);
String html = Http.url(url).get().html(); String html = Http.url(url).get().html();
String jsonString = html.substring(html.indexOf("var flashVars = {d: '") + 21); String jsonString = html.substring(html.indexOf("var flashVars = {d: '") + 21);
@@ -71,7 +73,7 @@ public class CliphunterRipper extends VideoRipper {
vidURL += c; vidURL += c;
} }
} }
addURLToDownload(new URL(vidURL), HOST + "_" + getGID(this.url)); addURLToDownload(new URI(vidURL).toURL(), HOST + "_" + getGID(this.url));
waitForThreads(); waitForThreads();
} }
} }

View File

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
@@ -56,7 +58,7 @@ public class PornhubRipper extends VideoRipper {
} }
@Override @Override
public void rip() throws IOException { public void rip() throws IOException, URISyntaxException {
String vidUrl = ""; String vidUrl = "";
LOGGER.info(" Retrieving " + this.url.toExternalForm()); LOGGER.info(" Retrieving " + this.url.toExternalForm());
Document doc = Http.url(this.url).get(); Document doc = Http.url(this.url).get();
@@ -146,7 +148,7 @@ public class PornhubRipper extends VideoRipper {
if (vidUrl.equals("")) { if (vidUrl.equals("")) {
throw new IOException("Unable to find encrypted video URL at " + this.url); throw new IOException("Unable to find encrypted video URL at " + this.url);
} }
addURLToDownload(new URL(vidUrl), HOST + "_" + bestQuality + "p_" + getGID(this.url)); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + bestQuality + "p_" + getGID(this.url));
waitForThreads(); waitForThreads();
} }

View File

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@@ -32,11 +34,6 @@ public class StickyXXXRipper extends VideoRipper {
return m.matches(); return m.matches();
} }
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
}
@Override @Override
public String getGID(URL url) throws MalformedURLException { public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://.*stickyxxx\\.com(/)(.*)/$"); Pattern p = Pattern.compile("^https?://.*stickyxxx\\.com(/)(.*)/$");
@@ -52,7 +49,7 @@ public class StickyXXXRipper extends VideoRipper {
} }
@Override @Override
public void rip() throws IOException { public void rip() throws IOException, URISyntaxException {
LOGGER.info("Retrieving " + this.url); LOGGER.info("Retrieving " + this.url);
Document doc = Http.url(url).get(); Document doc = Http.url(url).get();
Elements videos = doc.select(".wp-video > video > source"); Elements videos = doc.select(".wp-video > video > source");
@@ -60,7 +57,7 @@ public class StickyXXXRipper extends VideoRipper {
throw new IOException("Could not find Embed code at " + url); throw new IOException("Could not find Embed code at " + url);
} }
String vidUrl = videos.attr("src"); String vidUrl = videos.attr("src");
addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url)); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url));
waitForThreads(); waitForThreads();
} }
} }

View File

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@@ -34,11 +36,6 @@ public class TwitchVideoRipper extends VideoRipper {
return m.matches(); return m.matches();
} }
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
}
@Override @Override
public String getGID(URL url) throws MalformedURLException { public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https://clips\\.twitch\\.tv/(.*)$"); Pattern p = Pattern.compile("^https://clips\\.twitch\\.tv/(.*)$");
@@ -54,7 +51,7 @@ public class TwitchVideoRipper extends VideoRipper {
} }
@Override @Override
public void rip() throws IOException { public void rip() throws IOException, URISyntaxException {
LOGGER.info("Retrieving " + this.url); LOGGER.info("Retrieving " + this.url);
Document doc = Http.url(url).get(); Document doc = Http.url(url).get();
@@ -72,7 +69,7 @@ public class TwitchVideoRipper extends VideoRipper {
Matcher m = p.matcher(element.data()); Matcher m = p.matcher(element.data());
if (m.find()){ if (m.find()){
String vidUrl = m.group(1); String vidUrl = m.group(1);
addURLToDownload(new URL(vidUrl), HOST + "_" + title); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + title);
} }
} }
waitForThreads(); waitForThreads();

View File

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@@ -52,7 +54,7 @@ public class ViddmeRipper extends VideoRipper {
} }
@Override @Override
public void rip() throws IOException { public void rip() throws IOException, URISyntaxException {
LOGGER.info(" Retrieving " + this.url.toExternalForm()); LOGGER.info(" Retrieving " + this.url.toExternalForm());
Document doc = Http.url(this.url).get(); Document doc = Http.url(this.url).get();
Elements videos = doc.select("meta[name=twitter:player:stream]"); Elements videos = doc.select("meta[name=twitter:player:stream]");
@@ -61,7 +63,7 @@ public class ViddmeRipper extends VideoRipper {
} }
String vidUrl = videos.first().attr("content"); String vidUrl = videos.first().attr("content");
vidUrl = vidUrl.replaceAll("&amp;", "&"); vidUrl = vidUrl.replaceAll("&amp;", "&");
addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url)); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url));
waitForThreads(); waitForThreads();
} }
} }

View File

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.util.List; import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
@@ -33,11 +35,6 @@ public class VidearnRipper extends VideoRipper {
return m.matches(); return m.matches();
} }
@Override
public URL sanitizeURL(URL url) throws MalformedURLException {
return url;
}
@Override @Override
public String getGID(URL url) throws MalformedURLException { public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://[wm.]*videarn\\.com/[a-zA-Z0-9\\-]+/([0-9]+).*$"); Pattern p = Pattern.compile("^https?://[wm.]*videarn\\.com/[a-zA-Z0-9\\-]+/([0-9]+).*$");
@@ -53,15 +50,15 @@ public class VidearnRipper extends VideoRipper {
} }
@Override @Override
public void rip() throws IOException { public void rip() throws IOException, URISyntaxException {
LOGGER.info("Retrieving " + this.url); LOGGER.info("Retrieving " + this.url);
Document doc = Http.url(url).get(); Document doc = Http.url(url).get();
List<String> mp4s = Utils.between(doc.html(), "file:\"", "\""); List<String> mp4s = Utils.between(doc.html(), "file:\"", "\"");
if (mp4s.isEmpty()) { if (mp4s.isEmpty()) {
throw new IOException("Could not find files at " + url); throw new IOException("Could not find files at " + url);
} }
String vidUrl = mp4s.get(0); String vidUrl = mp4s.getFirst();
addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url)); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url));
waitForThreads(); waitForThreads();
} }
} }

View File

@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@@ -51,10 +53,10 @@ public class VkRipper extends VideoRipper {
} }
@Override @Override
public void rip() throws IOException { public void rip() throws IOException, URISyntaxException {
LOGGER.info(" Retrieving " + this.url); LOGGER.info(" Retrieving " + this.url);
String videoURL = getVideoURLAtPage(this.url.toExternalForm()); String videoURL = getVideoURLAtPage(this.url.toExternalForm());
addURLToDownload(new URL(videoURL), HOST + "_" + getGID(this.url)); addURLToDownload(new URI(videoURL).toURL(), HOST + "_" + getGID(this.url));
waitForThreads(); waitForThreads();
} }

View File

@@ -7,9 +7,11 @@ import java.net.URISyntaxException;
import com.rarchives.ripme.ripper.rippers.video.StickyXXXRipper; import com.rarchives.ripme.ripper.rippers.video.StickyXXXRipper;
// import com.rarchives.ripme.tst.ripper.rippers.RippersTest; // import com.rarchives.ripme.tst.ripper.rippers.RippersTest;
import com.rarchives.ripme.utils.Utils; import com.rarchives.ripme.utils.Utils;
import org.junit.jupiter.api.Test;
public class StickyXXXRipperTest extends RippersTest { public class StickyXXXRipperTest extends RippersTest {
@Test
public void testStickyXXXVideo() throws IOException, URISyntaxException { public void testStickyXXXVideo() throws IOException, URISyntaxException {
// This test fails on the CI - possibly due to checking for a file before it's written - so we're skipping it // This test fails on the CI - possibly due to checking for a file before it's written - so we're skipping it
if (Utils.getConfigBoolean("test.run_flaky_tests", false)) { if (Utils.getConfigBoolean("test.run_flaky_tests", false)) {