mirror of
https://github.com/RipMeApp/ripme.git
synced 2025-08-18 11:41:21 +02:00
new URI instead of new URL.
This commit is contained in:
@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -50,7 +52,7 @@ public class ArtStationRipper extends AbstractJSONRipper {
|
|||||||
try {
|
try {
|
||||||
// groupData = Http.url(albumURL.getLocation()).getJSON();
|
// groupData = Http.url(albumURL.getLocation()).getJSON();
|
||||||
groupData = getJson(albumURL.getLocation());
|
groupData = getJson(albumURL.getLocation());
|
||||||
} catch (IOException e) {
|
} catch (IOException | URISyntaxException e) {
|
||||||
throw new MalformedURLException("Couldn't load JSON from " + albumURL.getLocation());
|
throw new MalformedURLException("Couldn't load JSON from " + albumURL.getLocation());
|
||||||
}
|
}
|
||||||
return groupData.getString("title");
|
return groupData.getString("title");
|
||||||
@@ -62,7 +64,7 @@ public class ArtStationRipper extends AbstractJSONRipper {
|
|||||||
try {
|
try {
|
||||||
// groupData = Http.url(userInfoURL).getJSON();
|
// groupData = Http.url(userInfoURL).getJSON();
|
||||||
groupData = getJson(userInfoURL);
|
groupData = getJson(userInfoURL);
|
||||||
} catch (IOException e) {
|
} catch (IOException | URISyntaxException e) {
|
||||||
throw new MalformedURLException("Couldn't load JSON from " + userInfoURL);
|
throw new MalformedURLException("Couldn't load JSON from " + userInfoURL);
|
||||||
}
|
}
|
||||||
return groupData.getString("full_name");
|
return groupData.getString("full_name");
|
||||||
@@ -74,7 +76,7 @@ public class ArtStationRipper extends AbstractJSONRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected JSONObject getFirstPage() throws IOException {
|
protected JSONObject getFirstPage() throws IOException, URISyntaxException {
|
||||||
if (albumURL.getType() == URL_TYPE.SINGLE_PROJECT) {
|
if (albumURL.getType() == URL_TYPE.SINGLE_PROJECT) {
|
||||||
// URL points to JSON of a single project, just return it
|
// URL points to JSON of a single project, just return it
|
||||||
// return Http.url(albumURL.getLocation()).getJSON();
|
// return Http.url(albumURL.getLocation()).getJSON();
|
||||||
@@ -90,7 +92,7 @@ public class ArtStationRipper extends AbstractJSONRipper {
|
|||||||
if (albumContent.getInt("total_count") > 0) {
|
if (albumContent.getInt("total_count") > 0) {
|
||||||
// Get JSON of the first project and return it
|
// Get JSON of the first project and return it
|
||||||
JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(0);
|
JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(0);
|
||||||
ParsedURL projectURL = parseURL(new URL(projectInfo.getString("permalink")));
|
ParsedURL projectURL = parseURL(new URI(projectInfo.getString("permalink")).toURL());
|
||||||
// return Http.url(projectURL.getLocation()).getJSON();
|
// return Http.url(projectURL.getLocation()).getJSON();
|
||||||
return getJson(projectURL.getLocation());
|
return getJson(projectURL.getLocation());
|
||||||
}
|
}
|
||||||
@@ -100,7 +102,7 @@ public class ArtStationRipper extends AbstractJSONRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected JSONObject getNextPage(JSONObject doc) throws IOException {
|
protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException {
|
||||||
if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) {
|
if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) {
|
||||||
// Initialize the page number if it hasn't been initialized already
|
// Initialize the page number if it hasn't been initialized already
|
||||||
if (projectPageNumber == null) {
|
if (projectPageNumber == null) {
|
||||||
@@ -117,7 +119,7 @@ public class ArtStationRipper extends AbstractJSONRipper {
|
|||||||
projectIndex = 0;
|
projectIndex = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
Integer currentProject = ((projectPageNumber - 1) * 50) + (projectIndex + 1);
|
int currentProject = ((projectPageNumber - 1) * 50) + (projectIndex + 1);
|
||||||
// JSONObject albumContent = Http.url(albumURL.getLocation() + "?page=" +
|
// JSONObject albumContent = Http.url(albumURL.getLocation() + "?page=" +
|
||||||
// projectPageNumber).getJSON();
|
// projectPageNumber).getJSON();
|
||||||
JSONObject albumContent = getJson(albumURL.getLocation() + "?page=" + projectPageNumber);
|
JSONObject albumContent = getJson(albumURL.getLocation() + "?page=" + projectPageNumber);
|
||||||
@@ -125,7 +127,7 @@ public class ArtStationRipper extends AbstractJSONRipper {
|
|||||||
if (albumContent.getInt("total_count") > currentProject) {
|
if (albumContent.getInt("total_count") > currentProject) {
|
||||||
// Get JSON of the next project and return it
|
// Get JSON of the next project and return it
|
||||||
JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(projectIndex);
|
JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(projectIndex);
|
||||||
ParsedURL projectURL = parseURL(new URL(projectInfo.getString("permalink")));
|
ParsedURL projectURL = parseURL(new URI(projectInfo.getString("permalink")).toURL());
|
||||||
projectIndex++;
|
projectIndex++;
|
||||||
// return Http.url(projectURL.getLocation()).getJSON();
|
// return Http.url(projectURL.getLocation()).getJSON();
|
||||||
return getJson(projectURL.getLocation());
|
return getJson(projectURL.getLocation());
|
||||||
@@ -320,8 +322,8 @@ public class ArtStationRipper extends AbstractJSONRipper {
|
|||||||
throw new IOException("Error fetching json. Status code:" + status);
|
throw new IOException("Error fetching json. Status code:" + status);
|
||||||
}
|
}
|
||||||
|
|
||||||
private JSONObject getJson(String url) throws IOException {
|
private JSONObject getJson(String url) throws IOException, URISyntaxException {
|
||||||
return getJson(new URL(url));
|
return getJson(new URI(url).toURL());
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@@ -72,7 +72,7 @@ public class ChanRipper extends AbstractHTMLRipper {
|
|||||||
);
|
);
|
||||||
|
|
||||||
private ChanSite chanSite;
|
private ChanSite chanSite;
|
||||||
private Boolean generalChanSite = true;
|
private boolean generalChanSite = true;
|
||||||
|
|
||||||
public ChanRipper(URL url) throws IOException {
|
public ChanRipper(URL url) throws IOException {
|
||||||
super(url);
|
super(url);
|
||||||
|
@@ -94,13 +94,6 @@ public class EHentaiRipper extends AbstractHTMLRipper {
|
|||||||
+ " Got: " + url);
|
+ " Got: " + url);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Attempts to get page, checks for IP ban, waits.
|
|
||||||
*
|
|
||||||
* @param url
|
|
||||||
* @return Page document
|
|
||||||
* @throws IOException If page loading errors, or if retries are exhausted
|
|
||||||
*/
|
|
||||||
private Document getPageWithRetries(URL url) throws IOException {
|
private Document getPageWithRetries(URL url) throws IOException {
|
||||||
Document doc;
|
Document doc;
|
||||||
int retries = 3;
|
int retries = 3;
|
||||||
@@ -251,16 +244,16 @@ public class EHentaiRipper extends AbstractHTMLRipper {
|
|||||||
savePath += String.format("%03d_", index);
|
savePath += String.format("%03d_", index);
|
||||||
}
|
}
|
||||||
savePath += m.group(1);
|
savePath += m.group(1);
|
||||||
addURLToDownload(new URL(imgsrc), Paths.get(savePath));
|
addURLToDownload(new URI(imgsrc).toURL(), Paths.get(savePath));
|
||||||
} else {
|
} else {
|
||||||
// Provide prefix and let the AbstractRipper "guess" the filename
|
// Provide prefix and let the AbstractRipper "guess" the filename
|
||||||
String prefix = "";
|
String prefix = "";
|
||||||
if (Utils.getConfigBoolean("download.save_order", true)) {
|
if (Utils.getConfigBoolean("download.save_order", true)) {
|
||||||
prefix = String.format("%03d_", index);
|
prefix = String.format("%03d_", index);
|
||||||
}
|
}
|
||||||
addURLToDownload(new URL(imgsrc), prefix);
|
addURLToDownload(new URI(imgsrc).toURL(), prefix);
|
||||||
}
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException | URISyntaxException e) {
|
||||||
LOGGER.error("[!] Exception while loading/parsing " + this.url, e);
|
LOGGER.error("[!] Exception while loading/parsing " + this.url, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URI;
|
||||||
import java.net.URISyntaxException;
|
import java.net.URISyntaxException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
@@ -85,8 +86,8 @@ public class EromeRipper extends AbstractHTMLRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
|
||||||
return new URL(url.toExternalForm().replaceAll("https?://erome.com", "https://www.erome.com"));
|
return new URI(url.toExternalForm().replaceAll("https?://erome.com", "https://www.erome.com")).toURL();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@@ -1,9 +1,7 @@
|
|||||||
package com.rarchives.ripme.ripper.rippers;
|
package com.rarchives.ripme.ripper.rippers;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.HttpURLConnection;
|
import java.net.*;
|
||||||
import java.net.MalformedURLException;
|
|
||||||
import java.net.URL;
|
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.nio.file.Paths;
|
import java.nio.file.Paths;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
@@ -164,8 +162,8 @@ public class FivehundredpxRipper extends AbstractJSONRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public JSONObject getFirstPage() throws IOException {
|
public JSONObject getFirstPage() throws IOException, URISyntaxException {
|
||||||
URL apiURL = new URL(baseURL + "&consumer_key=" + CONSUMER_KEY);
|
URL apiURL = new URI(baseURL + "&consumer_key=" + CONSUMER_KEY).toURL();
|
||||||
LOGGER.debug("apiURL: " + apiURL);
|
LOGGER.debug("apiURL: " + apiURL);
|
||||||
JSONObject json = Http.url(apiURL).getJSON();
|
JSONObject json = Http.url(apiURL).getJSON();
|
||||||
|
|
||||||
@@ -232,7 +230,7 @@ public class FivehundredpxRipper extends AbstractJSONRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public JSONObject getNextPage(JSONObject json) throws IOException {
|
public JSONObject getNextPage(JSONObject json) throws IOException, URISyntaxException {
|
||||||
if (isThisATest()) {
|
if (isThisATest()) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -249,9 +247,9 @@ public class FivehundredpxRipper extends AbstractJSONRipper {
|
|||||||
|
|
||||||
sleep(500);
|
sleep(500);
|
||||||
++page;
|
++page;
|
||||||
URL apiURL = new URL(baseURL
|
URL apiURL = new URI(baseURL
|
||||||
+ "&page=" + page
|
+ "&page=" + page
|
||||||
+ "&consumer_key=" + CONSUMER_KEY);
|
+ "&consumer_key=" + CONSUMER_KEY).toURL();
|
||||||
return Http.url(apiURL).getJSON();
|
return Http.url(apiURL).getJSON();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -296,14 +294,9 @@ public class FivehundredpxRipper extends AbstractJSONRipper {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (imageURL == null) {
|
imageURLs.add(imageURL);
|
||||||
LOGGER.error("Failed to find image for photo " + photo.toString());
|
if (isThisATest()) {
|
||||||
}
|
break;
|
||||||
else {
|
|
||||||
imageURLs.add(imageURL);
|
|
||||||
if (isThisATest()) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return imageURLs;
|
return imageURLs;
|
||||||
@@ -311,13 +304,13 @@ public class FivehundredpxRipper extends AbstractJSONRipper {
|
|||||||
|
|
||||||
private boolean urlExists(String url) {
|
private boolean urlExists(String url) {
|
||||||
try {
|
try {
|
||||||
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
|
HttpURLConnection connection = (HttpURLConnection) new URI(url).toURL().openConnection();
|
||||||
connection.setRequestMethod("HEAD");
|
connection.setRequestMethod("HEAD");
|
||||||
if (connection.getResponseCode() != 200) {
|
if (connection.getResponseCode() != 200) {
|
||||||
throw new IOException("Couldn't find full-size image at " + url);
|
throw new IOException("Couldn't find full-size image at " + url);
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
} catch (IOException e) {
|
} catch (IOException | URISyntaxException e) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -40,7 +42,7 @@ public class FuskatorRipper extends AbstractHTMLRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
|
||||||
String u = url.toExternalForm();
|
String u = url.toExternalForm();
|
||||||
if (u.contains("/thumbs/")) {
|
if (u.contains("/thumbs/")) {
|
||||||
u = u.replace("/thumbs/", "/full/");
|
u = u.replace("/thumbs/", "/full/");
|
||||||
@@ -48,7 +50,7 @@ public class FuskatorRipper extends AbstractHTMLRipper {
|
|||||||
if (u.contains("/expanded/")) {
|
if (u.contains("/expanded/")) {
|
||||||
u = u.replaceAll("/expanded/", "/full/");
|
u = u.replaceAll("/expanded/", "/full/");
|
||||||
}
|
}
|
||||||
return new URL(u);
|
return new URI(u).toURL();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Base64;
|
import java.util.Base64;
|
||||||
@@ -80,16 +82,15 @@ public class HentaiNexusRipper extends AbstractJSONRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected JSONObject getFirstPage() throws IOException {
|
protected JSONObject getFirstPage() throws IOException, URISyntaxException {
|
||||||
String jsonEncodedString = getJsonEncodedStringFromPage();
|
String jsonEncodedString = getJsonEncodedStringFromPage();
|
||||||
String jsonDecodedString = decodeJsonString(jsonEncodedString);
|
String jsonDecodedString = decodeJsonString(jsonEncodedString);
|
||||||
return new JSONObject(jsonDecodedString);
|
return new JSONObject(jsonDecodedString);
|
||||||
}
|
}
|
||||||
|
|
||||||
public String getJsonEncodedStringFromPage() throws MalformedURLException, IOException
|
public String getJsonEncodedStringFromPage() throws MalformedURLException, IOException, URISyntaxException {
|
||||||
{
|
|
||||||
// Image data only appears on the /read/ page and not on the /view/ one.
|
// Image data only appears on the /read/ page and not on the /view/ one.
|
||||||
URL readUrl = new URL(String.format("http://hentainexus.com/read/%s",getGID(url)));
|
URL readUrl = new URI(String.format("http://hentainexus.com/read/%s",getGID(url))).toURL();
|
||||||
Document document = Http.url(readUrl).response().parse();
|
Document document = Http.url(readUrl).response().parse();
|
||||||
|
|
||||||
for (Element scripts : document.getElementsByTag("script")) {
|
for (Element scripts : document.getElementsByTag("script")) {
|
||||||
@@ -143,7 +144,7 @@ public class HentaiNexusRipper extends AbstractJSONRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
magicByte = (byte) (magicByte & 0x7);
|
magicByte = (byte) (magicByte & 0x7);
|
||||||
ArrayList<Integer> newArray = new ArrayList();
|
ArrayList<Integer> newArray = new ArrayList<>();
|
||||||
|
|
||||||
for (int i = 0x0; i < 0x100; i++) {
|
for (int i = 0x0; i < 0x100; i++) {
|
||||||
newArray.add(i);
|
newArray.add(i);
|
||||||
|
@@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URI;
|
||||||
import java.net.URISyntaxException;
|
import java.net.URISyntaxException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
@@ -14,7 +15,6 @@ import org.jsoup.nodes.Document;
|
|||||||
|
|
||||||
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
|
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
|
||||||
import com.rarchives.ripme.utils.Http;
|
import com.rarchives.ripme.utils.Http;
|
||||||
import org.jsoup.nodes.Element;
|
|
||||||
|
|
||||||
public class HitomiRipper extends AbstractHTMLRipper {
|
public class HitomiRipper extends AbstractHTMLRipper {
|
||||||
|
|
||||||
@@ -47,9 +47,9 @@ public class HitomiRipper extends AbstractHTMLRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Document getFirstPage() throws IOException {
|
public Document getFirstPage() throws IOException, URISyntaxException {
|
||||||
// if we go to /GALLERYID.js we get a nice json array of all images in the gallery
|
// if we go to /GALLERYID.js we get a nice json array of all images in the gallery
|
||||||
return Http.url(new URL(url.toExternalForm().replaceAll("hitomi", "ltn.hitomi").replaceAll(".html", ".js"))).ignoreContentType().get();
|
return Http.url(new URI(url.toExternalForm().replaceAll("hitomi", "ltn.hitomi").replaceAll(".html", ".js")).toURL()).ignoreContentType().get();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@@ -8,6 +8,8 @@ import org.json.JSONObject;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -34,7 +36,7 @@ public class NsfwXxxRipper extends AbstractJSONRipper {
|
|||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
|
||||||
String u = url.toExternalForm();
|
String u = url.toExternalForm();
|
||||||
// https://nsfw.xxx/user/kelly-kat/foo -> https://nsfw.xxx/user/kelly-kat
|
// https://nsfw.xxx/user/kelly-kat/foo -> https://nsfw.xxx/user/kelly-kat
|
||||||
// https://nsfw.xxx/user/kelly-kat -> https://nsfw.xxx/user/kelly-kat
|
// https://nsfw.xxx/user/kelly-kat -> https://nsfw.xxx/user/kelly-kat
|
||||||
@@ -44,15 +46,15 @@ public class NsfwXxxRipper extends AbstractJSONRipper {
|
|||||||
throw new MalformedURLException("Invalid URL: " + url);
|
throw new MalformedURLException("Invalid URL: " + url);
|
||||||
}
|
}
|
||||||
|
|
||||||
return new URL(u);
|
return new URI(u).toURL();
|
||||||
}
|
}
|
||||||
|
|
||||||
String getUser() throws MalformedURLException {
|
String getUser() throws MalformedURLException {
|
||||||
return getGID(url);
|
return getGID(url);
|
||||||
}
|
}
|
||||||
|
|
||||||
URL getPage(int page) throws MalformedURLException {
|
URL getPage(int page) throws MalformedURLException, URISyntaxException {
|
||||||
return new URL("https://nsfw.xxx/slide-page/" + page + "?nsfw%5B%5D=0&types%5B%5D=image&types%5B%5D=video&types%5B%5D=gallery&slider=1&jsload=1&user=" + getUser());
|
return new URI("https://nsfw.xxx/slide-page/" + page + "?nsfw%5B%5D=0&types%5B%5D=image&types%5B%5D=video&types%5B%5D=gallery&slider=1&jsload=1&user=" + getUser()).toURL();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -71,18 +73,18 @@ public class NsfwXxxRipper extends AbstractJSONRipper {
|
|||||||
int currentPage = 1;
|
int currentPage = 1;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected JSONObject getFirstPage() throws IOException {
|
protected JSONObject getFirstPage() throws IOException, URISyntaxException {
|
||||||
return Http.url(getPage(1)).getJSON();
|
return Http.url(getPage(1)).getJSON();
|
||||||
}
|
}
|
||||||
|
|
||||||
List<String> descriptions = new ArrayList<>();
|
List<String> descriptions = new ArrayList<>();
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected JSONObject getNextPage(JSONObject doc) throws IOException {
|
protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException {
|
||||||
currentPage++;
|
currentPage++;
|
||||||
JSONObject nextPage = Http.url(getPage(doc.getInt("page") + 1)).getJSON();
|
JSONObject nextPage = Http.url(getPage(doc.getInt("page") + 1)).getJSON();
|
||||||
JSONArray items = nextPage.getJSONArray("items");
|
JSONArray items = nextPage.getJSONArray("items");
|
||||||
if (items.length() == 0) {
|
if (items.isEmpty()) {
|
||||||
throw new IOException("No more pages");
|
throw new IOException("No more pages");
|
||||||
}
|
}
|
||||||
return nextPage;
|
return nextPage;
|
||||||
@@ -120,7 +122,7 @@ public class NsfwXxxRipper extends AbstractJSONRipper {
|
|||||||
|
|
||||||
return new ApiEntry(srcUrl, o.getString("author"), o.getString("title"));
|
return new ApiEntry(srcUrl, o.getString("author"), o.getString("title"));
|
||||||
})
|
})
|
||||||
.collect(Collectors.toList());
|
.toList();
|
||||||
|
|
||||||
data.forEach(e -> descriptions.add(e.title));
|
data.forEach(e -> descriptions.add(e.title));
|
||||||
return data.stream().map(e -> e.srcUrl).collect(Collectors.toList());
|
return data.stream().map(e -> e.srcUrl).collect(Collectors.toList());
|
||||||
|
@@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URI;
|
||||||
import java.net.URISyntaxException;
|
import java.net.URISyntaxException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
@@ -44,13 +45,13 @@ public class XhamsterRipper extends AbstractHTMLRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
|
||||||
if (isVideoUrl(url)) {
|
if (isVideoUrl(url)) {
|
||||||
return url;
|
return url;
|
||||||
}
|
}
|
||||||
String URLToReturn = url.toExternalForm();
|
String URLToReturn = url.toExternalForm();
|
||||||
URLToReturn = URLToReturn.replaceAll("https?://\\w?\\w?\\.?xhamster([^<]*)\\.", "https://m.xhamster$1.");
|
URLToReturn = URLToReturn.replaceAll("https?://\\w?\\w?\\.?xhamster([^<]*)\\.", "https://m.xhamster$1.");
|
||||||
URL san_url = new URL(URLToReturn);
|
URL san_url = new URI(URLToReturn).toURL();
|
||||||
LOGGER.info("sanitized URL is " + san_url.toExternalForm());
|
LOGGER.info("sanitized URL is " + san_url.toExternalForm());
|
||||||
return san_url;
|
return san_url;
|
||||||
}
|
}
|
||||||
@@ -168,10 +169,10 @@ public class XhamsterRipper extends AbstractHTMLRipper {
|
|||||||
// This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to
|
// This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to
|
||||||
// the page chamster.com but displays the mobile site from m.xhamster.com
|
// the page chamster.com but displays the mobile site from m.xhamster.com
|
||||||
pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster([^<]*)\\.", "://m.xhamster$1.");
|
pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster([^<]*)\\.", "://m.xhamster$1.");
|
||||||
String image = Http.url(new URL(pageWithImageUrl)).get().select("a > img#photoCurr").attr("src");
|
String image = Http.url(new URI(pageWithImageUrl).toURL()).get().select("a > img#photoCurr").attr("src");
|
||||||
result.add(image);
|
result.add(image);
|
||||||
downloadFile(image);
|
downloadFile(image);
|
||||||
} catch (IOException e) {
|
} catch (IOException | URISyntaxException e) {
|
||||||
LOGGER.error("Was unable to load page " + pageWithImageUrl);
|
LOGGER.error("Was unable to load page " + pageWithImageUrl);
|
||||||
}
|
}
|
||||||
if (isStopped() || isThisATest()) {
|
if (isStopped() || isThisATest()) {
|
||||||
|
@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.util.regex.Matcher;
|
import java.util.regex.Matcher;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
@@ -54,7 +56,7 @@ public class CliphunterRipper extends VideoRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void rip() throws IOException {
|
public void rip() throws IOException, URISyntaxException {
|
||||||
LOGGER.info("Retrieving " + this.url);
|
LOGGER.info("Retrieving " + this.url);
|
||||||
String html = Http.url(url).get().html();
|
String html = Http.url(url).get().html();
|
||||||
String jsonString = html.substring(html.indexOf("var flashVars = {d: '") + 21);
|
String jsonString = html.substring(html.indexOf("var flashVars = {d: '") + 21);
|
||||||
@@ -71,7 +73,7 @@ public class CliphunterRipper extends VideoRipper {
|
|||||||
vidURL += c;
|
vidURL += c;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
addURLToDownload(new URL(vidURL), HOST + "_" + getGID(this.url));
|
addURLToDownload(new URI(vidURL).toURL(), HOST + "_" + getGID(this.url));
|
||||||
waitForThreads();
|
waitForThreads();
|
||||||
}
|
}
|
||||||
}
|
}
|
@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
@@ -56,7 +58,7 @@ public class PornhubRipper extends VideoRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void rip() throws IOException {
|
public void rip() throws IOException, URISyntaxException {
|
||||||
String vidUrl = "";
|
String vidUrl = "";
|
||||||
LOGGER.info(" Retrieving " + this.url.toExternalForm());
|
LOGGER.info(" Retrieving " + this.url.toExternalForm());
|
||||||
Document doc = Http.url(this.url).get();
|
Document doc = Http.url(this.url).get();
|
||||||
@@ -146,7 +148,7 @@ public class PornhubRipper extends VideoRipper {
|
|||||||
if (vidUrl.equals("")) {
|
if (vidUrl.equals("")) {
|
||||||
throw new IOException("Unable to find encrypted video URL at " + this.url);
|
throw new IOException("Unable to find encrypted video URL at " + this.url);
|
||||||
}
|
}
|
||||||
addURLToDownload(new URL(vidUrl), HOST + "_" + bestQuality + "p_" + getGID(this.url));
|
addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + bestQuality + "p_" + getGID(this.url));
|
||||||
|
|
||||||
waitForThreads();
|
waitForThreads();
|
||||||
}
|
}
|
||||||
|
@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.util.regex.Matcher;
|
import java.util.regex.Matcher;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
@@ -32,11 +34,6 @@ public class StickyXXXRipper extends VideoRipper {
|
|||||||
return m.matches();
|
return m.matches();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
|
||||||
return url;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getGID(URL url) throws MalformedURLException {
|
public String getGID(URL url) throws MalformedURLException {
|
||||||
Pattern p = Pattern.compile("^https?://.*stickyxxx\\.com(/)(.*)/$");
|
Pattern p = Pattern.compile("^https?://.*stickyxxx\\.com(/)(.*)/$");
|
||||||
@@ -52,7 +49,7 @@ public class StickyXXXRipper extends VideoRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void rip() throws IOException {
|
public void rip() throws IOException, URISyntaxException {
|
||||||
LOGGER.info("Retrieving " + this.url);
|
LOGGER.info("Retrieving " + this.url);
|
||||||
Document doc = Http.url(url).get();
|
Document doc = Http.url(url).get();
|
||||||
Elements videos = doc.select(".wp-video > video > source");
|
Elements videos = doc.select(".wp-video > video > source");
|
||||||
@@ -60,7 +57,7 @@ public class StickyXXXRipper extends VideoRipper {
|
|||||||
throw new IOException("Could not find Embed code at " + url);
|
throw new IOException("Could not find Embed code at " + url);
|
||||||
}
|
}
|
||||||
String vidUrl = videos.attr("src");
|
String vidUrl = videos.attr("src");
|
||||||
addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url));
|
addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url));
|
||||||
waitForThreads();
|
waitForThreads();
|
||||||
}
|
}
|
||||||
}
|
}
|
@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.util.regex.Matcher;
|
import java.util.regex.Matcher;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
@@ -34,11 +36,6 @@ public class TwitchVideoRipper extends VideoRipper {
|
|||||||
return m.matches();
|
return m.matches();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
|
||||||
return url;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getGID(URL url) throws MalformedURLException {
|
public String getGID(URL url) throws MalformedURLException {
|
||||||
Pattern p = Pattern.compile("^https://clips\\.twitch\\.tv/(.*)$");
|
Pattern p = Pattern.compile("^https://clips\\.twitch\\.tv/(.*)$");
|
||||||
@@ -54,7 +51,7 @@ public class TwitchVideoRipper extends VideoRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void rip() throws IOException {
|
public void rip() throws IOException, URISyntaxException {
|
||||||
LOGGER.info("Retrieving " + this.url);
|
LOGGER.info("Retrieving " + this.url);
|
||||||
Document doc = Http.url(url).get();
|
Document doc = Http.url(url).get();
|
||||||
|
|
||||||
@@ -72,7 +69,7 @@ public class TwitchVideoRipper extends VideoRipper {
|
|||||||
Matcher m = p.matcher(element.data());
|
Matcher m = p.matcher(element.data());
|
||||||
if (m.find()){
|
if (m.find()){
|
||||||
String vidUrl = m.group(1);
|
String vidUrl = m.group(1);
|
||||||
addURLToDownload(new URL(vidUrl), HOST + "_" + title);
|
addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + title);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
waitForThreads();
|
waitForThreads();
|
||||||
|
@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.util.regex.Matcher;
|
import java.util.regex.Matcher;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
@@ -52,7 +54,7 @@ public class ViddmeRipper extends VideoRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void rip() throws IOException {
|
public void rip() throws IOException, URISyntaxException {
|
||||||
LOGGER.info(" Retrieving " + this.url.toExternalForm());
|
LOGGER.info(" Retrieving " + this.url.toExternalForm());
|
||||||
Document doc = Http.url(this.url).get();
|
Document doc = Http.url(this.url).get();
|
||||||
Elements videos = doc.select("meta[name=twitter:player:stream]");
|
Elements videos = doc.select("meta[name=twitter:player:stream]");
|
||||||
@@ -61,7 +63,7 @@ public class ViddmeRipper extends VideoRipper {
|
|||||||
}
|
}
|
||||||
String vidUrl = videos.first().attr("content");
|
String vidUrl = videos.first().attr("content");
|
||||||
vidUrl = vidUrl.replaceAll("&", "&");
|
vidUrl = vidUrl.replaceAll("&", "&");
|
||||||
addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url));
|
addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url));
|
||||||
waitForThreads();
|
waitForThreads();
|
||||||
}
|
}
|
||||||
}
|
}
|
@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.regex.Matcher;
|
import java.util.regex.Matcher;
|
||||||
@@ -33,11 +35,6 @@ public class VidearnRipper extends VideoRipper {
|
|||||||
return m.matches();
|
return m.matches();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
|
||||||
return url;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getGID(URL url) throws MalformedURLException {
|
public String getGID(URL url) throws MalformedURLException {
|
||||||
Pattern p = Pattern.compile("^https?://[wm.]*videarn\\.com/[a-zA-Z0-9\\-]+/([0-9]+).*$");
|
Pattern p = Pattern.compile("^https?://[wm.]*videarn\\.com/[a-zA-Z0-9\\-]+/([0-9]+).*$");
|
||||||
@@ -53,15 +50,15 @@ public class VidearnRipper extends VideoRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void rip() throws IOException {
|
public void rip() throws IOException, URISyntaxException {
|
||||||
LOGGER.info("Retrieving " + this.url);
|
LOGGER.info("Retrieving " + this.url);
|
||||||
Document doc = Http.url(url).get();
|
Document doc = Http.url(url).get();
|
||||||
List<String> mp4s = Utils.between(doc.html(), "file:\"", "\"");
|
List<String> mp4s = Utils.between(doc.html(), "file:\"", "\"");
|
||||||
if (mp4s.isEmpty()) {
|
if (mp4s.isEmpty()) {
|
||||||
throw new IOException("Could not find files at " + url);
|
throw new IOException("Could not find files at " + url);
|
||||||
}
|
}
|
||||||
String vidUrl = mp4s.get(0);
|
String vidUrl = mp4s.getFirst();
|
||||||
addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url));
|
addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url));
|
||||||
waitForThreads();
|
waitForThreads();
|
||||||
}
|
}
|
||||||
}
|
}
|
@@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video;
|
|||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.util.regex.Matcher;
|
import java.util.regex.Matcher;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
@@ -51,10 +53,10 @@ public class VkRipper extends VideoRipper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void rip() throws IOException {
|
public void rip() throws IOException, URISyntaxException {
|
||||||
LOGGER.info(" Retrieving " + this.url);
|
LOGGER.info(" Retrieving " + this.url);
|
||||||
String videoURL = getVideoURLAtPage(this.url.toExternalForm());
|
String videoURL = getVideoURLAtPage(this.url.toExternalForm());
|
||||||
addURLToDownload(new URL(videoURL), HOST + "_" + getGID(this.url));
|
addURLToDownload(new URI(videoURL).toURL(), HOST + "_" + getGID(this.url));
|
||||||
waitForThreads();
|
waitForThreads();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@@ -7,9 +7,11 @@ import java.net.URISyntaxException;
|
|||||||
import com.rarchives.ripme.ripper.rippers.video.StickyXXXRipper;
|
import com.rarchives.ripme.ripper.rippers.video.StickyXXXRipper;
|
||||||
// import com.rarchives.ripme.tst.ripper.rippers.RippersTest;
|
// import com.rarchives.ripme.tst.ripper.rippers.RippersTest;
|
||||||
import com.rarchives.ripme.utils.Utils;
|
import com.rarchives.ripme.utils.Utils;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
public class StickyXXXRipperTest extends RippersTest {
|
public class StickyXXXRipperTest extends RippersTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
public void testStickyXXXVideo() throws IOException, URISyntaxException {
|
public void testStickyXXXVideo() throws IOException, URISyntaxException {
|
||||||
// This test fails on the CI - possibly due to checking for a file before it's written - so we're skipping it
|
// This test fails on the CI - possibly due to checking for a file before it's written - so we're skipping it
|
||||||
if (Utils.getConfigBoolean("test.run_flaky_tests", false)) {
|
if (Utils.getConfigBoolean("test.run_flaky_tests", false)) {
|
||||||
|
Reference in New Issue
Block a user