1
0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-01-17 12:48:24 +01:00

new URI instead of new URL, rule34, getFirstPage.

This commit is contained in:
soloturn 2023-12-29 11:32:04 +01:00
parent f5153de8cf
commit c3a4df47e9
34 changed files with 62 additions and 42 deletions

View File

@ -41,11 +41,11 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
protected abstract String getDomain();
public abstract String getHost();
protected Document getFirstPage() throws IOException {
protected Document getFirstPage() throws IOException, URISyntaxException {
return Http.url(url).get();
}
protected Document getCachedFirstPage() throws IOException {
protected Document getCachedFirstPage() throws IOException, URISyntaxException {
if (cachedFirstPage == null) {
cachedFirstPage = getFirstPage();
}
@ -462,7 +462,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
* URL to define how the working directory should be saved.
*/
@Override
public void setWorkingDir(URL url) throws IOException {
public void setWorkingDir(URL url) throws IOException, URISyntaxException {
Path wd = Utils.getWorkingDirectory();
// TODO - change to nio
String path = wd.toAbsolutePath().toString();

View File

@ -283,7 +283,7 @@ public abstract class AbstractJSONRipper extends AbstractRipper {
* IOException
*/
@Override
public void setWorkingDir(URL url) throws IOException {
public void setWorkingDir(URL url) throws IOException, URISyntaxException {
Path wd = Utils.getWorkingDirectory();
String title;
if (Utils.getConfigBoolean("album_titles.save", true)) {

View File

@ -188,7 +188,7 @@ public abstract class AbstractRipper
* @throws IOException
* Always be prepared.
*/
public void setup() throws IOException {
public void setup() throws IOException, URISyntaxException {
setWorkingDir(this.url);
// we do not care if the rollingfileappender is active, just change the logfile in case
// TODO this does not work - not even with
@ -537,7 +537,7 @@ public abstract class AbstractRipper
}
@Override
public abstract void setWorkingDir(URL url) throws IOException;
public abstract void setWorkingDir(URL url) throws IOException, URISyntaxException;
/**
*
@ -550,7 +550,7 @@ public abstract class AbstractRipper
* @throws MalformedURLException
* If any of those damned URLs gets malformed.
*/
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
try {
return getHost() + "_" + getGID(url);
} catch (URISyntaxException e) {

View File

@ -192,7 +192,7 @@ public abstract class AlbumRipper extends AbstractRipper {
* IOException
*/
@Override
public void setWorkingDir(URL url) throws IOException {
public void setWorkingDir(URL url) throws IOException, URISyntaxException {
Path wd = Utils.getWorkingDirectory();
// TODO - change to nio
String path = wd.toAbsolutePath().toString();

View File

@ -15,7 +15,7 @@ interface RipperInterface {
void rip() throws IOException, URISyntaxException;
boolean canRip(URL url);
URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException;
void setWorkingDir(URL url) throws IOException;
void setWorkingDir(URL url) throws IOException, URISyntaxException;
String getHost();
String getGID(URL url) throws MalformedURLException, URISyntaxException;
}

View File

@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@ -46,7 +47,7 @@ public class AerisdiesRipper extends AbstractHTMLRipper {
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
try {
Element el = getCachedFirstPage().select(".headtext").first();
if (el == null) {

View File

@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@ -69,7 +70,7 @@ public class BatoRipper extends AbstractHTMLRipper {
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
try {
// Attempt to use album title as GID
return getHost() + "_" + getGID(url) + "_" + getCachedFirstPage().select("title").first().text().replaceAll(" ", "_");

View File

@ -6,6 +6,7 @@ import com.rarchives.ripme.utils.Utils;
import com.rarchives.ripme.utils.RipUtils;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
@ -194,7 +195,7 @@ public class ChanRipper extends AbstractHTMLRipper {
return this.url.getHost();
}
public Document getFirstPage() throws IOException {
public Document getFirstPage() throws IOException, URISyntaxException {
return super.getFirstPage();
}
private boolean isURLBlacklisted(String url) {

View File

@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
@ -50,7 +51,7 @@ public class CheveretoRipper extends AbstractHTMLRipper {
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
try {
// Attempt to use album title as GID
Element titleElement = getCachedFirstPage().select("meta[property=og:title]").first();

View File

@ -62,7 +62,7 @@ public class EHentaiRipper extends AbstractHTMLRipper {
return "e-hentai.org";
}
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
try {
// Attempt to use album title as GID
if (albumDoc == null) {

View File

@ -57,7 +57,7 @@ public class EightmusesRipper extends AbstractHTMLRipper {
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
try {
// Attempt to use album title as GID
Element titleElement = getCachedFirstPage().select("meta[name=description]").first();

View File

@ -95,7 +95,7 @@ public class EroShareRipper extends AbstractHTMLRipper {
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
if (!is_profile(url)) {
try {
// Attempt to use album title as GID

View File

@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
@ -67,7 +68,7 @@ public class EromeRipper extends AbstractHTMLRipper {
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
try {
// Attempt to use album title as GID
Element titleElement = getCachedFirstPage().select("meta[property=og:title]").first();

View File

@ -173,7 +173,7 @@ public class FlickrRipper extends AbstractHTMLRipper {
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
if (!url.toExternalForm().contains("/sets/")) {
return super.getAlbumTitle(url);
}

View File

@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@ -29,7 +30,7 @@ public class GirlsOfDesireRipper extends AbstractHTMLRipper {
return "girlsofdesire.org";
}
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
try {
// Attempt to use album title as GID
Document doc = getCachedFirstPage();

View File

@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@ -49,7 +50,7 @@ public class HbrowseRipper extends AbstractHTMLRipper {
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
try {
Document doc = getCachedFirstPage();
String title = doc.select("div[id=main] > table.listTable > tbody > tr > td.listLong").first().text();

View File

@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@ -90,7 +91,7 @@ public class Hentai2readRipper extends AbstractHTMLRipper {
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
try {
return getHost() + "_" + getGID(url);
} catch (Exception e) {

View File

@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@ -52,7 +53,7 @@ public class HentaifoxRipper extends AbstractHTMLRipper {
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
try {
Document doc = getCachedFirstPage();
String title = doc.select("div.info > h1").first().text();

View File

@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@ -65,7 +66,7 @@ public class HitomiRipper extends AbstractHTMLRipper {
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
try {
// Attempt to use album title and username as GID
Document doc = Http.url(url).get();

View File

@ -11,6 +11,7 @@ import org.jsoup.select.Elements;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@ -63,7 +64,7 @@ public class HqpornerRipper extends AbstractHTMLRipper {
}
@Override
public Document getFirstPage() throws IOException {
public Document getFirstPage() throws IOException, URISyntaxException {
return super.getFirstPage();
}

View File

@ -87,7 +87,7 @@ public class ImagebamRipper extends AbstractHTMLRipper {
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
try {
// Attempt to use album title as GID
Elements elems = getCachedFirstPage().select("[id=gallery-name]");

View File

@ -166,7 +166,7 @@ public class ImagefapRipper extends AbstractHTMLRipper {
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
try {
// Attempt to use album title as GID
String title = getCachedFirstPage().title();

View File

@ -4,6 +4,7 @@ import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@ -82,7 +83,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper {
}
@Override
public Document getFirstPage() throws IOException {
public Document getFirstPage() throws IOException, URISyntaxException {
return super.getFirstPage();
}

View File

@ -155,7 +155,7 @@ public class NfsfwRipper extends AbstractHTMLRipper {
List<String> imageURLs = getImagePageURLs(fstPage);
List<String> subalbumURLs = getSubalbumURLs(fstPage);
return imageURLs.isEmpty() && !subalbumURLs.isEmpty();
} catch (IOException e) {
} catch (IOException | URISyntaxException e) {
LOGGER.error("Unable to load " + url, e);
return false;
}

View File

@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@ -51,13 +53,13 @@ public class Rule34Ripper extends AbstractHTMLRipper {
"rule34.xxx/index.php?page=post&s=list&tags=TAG - got " + url + " instead");
}
public URL getAPIUrl() throws MalformedURLException {
URL urlToReturn = new URL("https://rule34.xxx/index.php?page=dapi&s=post&q=index&limit=100&tags=" + getGID(url));
public URL getAPIUrl() throws MalformedURLException, URISyntaxException {
URL urlToReturn = new URI("https://rule34.xxx/index.php?page=dapi&s=post&q=index&limit=100&tags=" + getGID(url)).toURL();
return urlToReturn;
}
@Override
public Document getFirstPage() throws IOException {
public Document getFirstPage() throws IOException, URISyntaxException {
apiUrl = getAPIUrl().toExternalForm();
// "url" is an instance field of the superclass
return Http.url(getAPIUrl()).get();

View File

@ -43,7 +43,7 @@ public class SoundgasmRipper extends AbstractHTMLRipper {
}
@Override
public Document getFirstPage() throws IOException {
public Document getFirstPage() throws IOException, URISyntaxException {
return super.getFirstPage();
}

View File

@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@ -31,7 +32,7 @@ public class ViewcomicRipper extends AbstractHTMLRipper {
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
try {
// Attempt to use album title as GID
String titleText = getCachedFirstPage().select("title").first().text();

View File

@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@ -43,7 +44,7 @@ public class WebtoonsRipper extends AbstractHTMLRipper {
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
Pattern pat = Pattern.compile("https?://www.webtoons.com/[a-zA-Z-_]+/[a-zA-Z_-]+/([a-zA-Z0-9_-]*)/[a-zA-Z0-9_-]+/\\S*");
Matcher mat = pat.matcher(url.toExternalForm());
if (mat.matches()) {

View File

@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
@ -220,7 +221,7 @@ public class WordpressComicRipper extends AbstractHTMLRipper {
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
Pattern totempole666Pat = Pattern.compile("(?:https?://)?(?:www\\.)?totempole666.com/comic/([a-zA-Z0-9_-]*)/?$");
Matcher totempole666Mat = totempole666Pat.matcher(url.toExternalForm());
if (totempole666Mat.matches()) {

View File

@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@ -146,7 +147,7 @@ public class XhamsterRipper extends AbstractHTMLRipper {
}
@Override
public Document getFirstPage() throws IOException {
public Document getFirstPage() throws IOException, URISyntaxException {
return super.getFirstPage();
}
@ -215,7 +216,7 @@ public class XhamsterRipper extends AbstractHTMLRipper {
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
try {
// Attempt to use album title and username as GID
Document doc = getCachedFirstPage();

View File

@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@ -104,7 +105,7 @@ public class XvideosRipper extends AbstractSingleFileRipper {
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/profiles/([a-zA-Z0-9_-]+)/photos/(\\d+)/([a-zA-Z0-9_-]+)$");
Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) {

View File

@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
@ -45,7 +46,7 @@ public class ZizkiRipper extends AbstractHTMLRipper {
}
@Override
public String getAlbumTitle(URL url) throws MalformedURLException {
public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException {
try {
// Attempt to use album title as GID
Element titleElement = getCachedFirstPage().select("h1.title").first();

View File

@ -1502,7 +1502,7 @@ public final class MainWindow implements Runnable, RipStatusHandler {
entry.count = rsc.count;
try {
entry.title = ripper.getAlbumTitle(ripper.getURL());
} catch (MalformedURLException e) {
} catch (MalformedURLException | URISyntaxException e) {
LOGGER.warn(e.getMessage());
}
HISTORY.add(entry);

View File

@ -37,7 +37,7 @@ public class ChanRipperTest extends RippersTest {
CompletableFuture<Void> setupFuture = CompletableFuture.runAsync(() -> {
try {
ripper.setup();
} catch (IOException e) {
} catch (IOException | URISyntaxException e) {
throw new RuntimeException(e);
}
});