From ac57cc37f9ed97fa9f98b2cc27e542808295a94d Mon Sep 17 00:00:00 2001 From: Destrae Date: Wed, 30 Oct 2019 11:25:38 -0500 Subject: [PATCH 001/512] Updated README.md header standardization and grammar --- README.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 661b6f87..dd6438f7 100644 --- a/README.md +++ b/README.md @@ -16,12 +16,11 @@ RipMe is maintained with ♥️ and in our limited free time by **[@MetaPrime](h # About -RipMe is an album ripper for various websites. Runs on your computer. Requires Java 8. -RipMe is a cross-platform tool. It has been tested and confirmed working on Windows, Linux and MacOS. +RipMe is an album ripper for various websites. It is a cross-platform tool that runs on your computer, and requires Java 8. RipMe has been tested and confirmed working on Windows, Linux and MacOS. ![Screenshot](https://i.imgur.com/UCQNjeg.png) -## [Downloads](https://github.com/ripmeapp/ripme/releases) +## Downloads Download `ripme.jar` from the [latest release](https://github.com/ripmeapp/ripme/releases). @@ -29,18 +28,20 @@ Download `ripme.jar` from the [latest release](https://github.com/ripmeapp/ripme For information about running the `.jar` file, see [the How To Run wiki](https://github.com/ripmeapp/ripme/wiki/How-To-Run-RipMe). -## [Changelog](https://github.com/ripmeapp/ripme/blob/master/ripme.json) (ripme.json) +## Changelog + +[Changelog](https://github.com/ripmeapp/ripme/blob/master/ripme.json) **(ripme.json)** # Features -* Quickly downloads all images in an online album (see supported sites below) +* Quickly downloads all images in an online album. [See supported sites](https://github.com/ripmeapp/ripme/wiki/Supported-Sites) * Easily re-rip albums to fetch new content * Built in updater * Skips already downloaded images by default -* Can auto skip e-hentai and nhentai albums containing certain tags [See here for how to enable](https://github.com/RipMeApp/ripme/wiki/Config-options#nhentaiblacklisttags) -* Download a range of urls [See here for how](https://github.com/RipMeApp/ripme/wiki/How-To-Run-RipMe#downloading-a-url-range) +* Can auto skip e-hentai and nhentai albums containing certain tags. [See here for how to enable](https://github.com/RipMeApp/ripme/wiki/Config-options#nhentaiblacklisttags) +* Download a range of urls. [See here for how](https://github.com/RipMeApp/ripme/wiki/How-To-Run-RipMe#downloading-a-url-range) -## [List of Supported Sites](https://github.com/ripmeapp/ripme/wiki/Supported-Sites) +## List of Supported Sites * imgur * twitter @@ -58,13 +59,13 @@ For information about running the `.jar` file, see [the How To Run wiki](https:/ * 8muses * deviantart * xhamster -* (more) +* [(more)](https://github.com/ripmeapp/ripme/wiki/Supported-Sites) ## Not Supported? Request support for more sites by adding a comment to [this Github issue](https://github.com/RipMeApp/ripme/issues/38). -If you're a developer, you can add your own Ripper by following the wiki guide +If you're a developer, you can add your own Ripper by following the wiki guide: [How To Create A Ripper for HTML Websites](https://github.com/ripmeapp/ripme/wiki/How-To-Create-A-Ripper-for-HTML-websites). # Compiling & Building From 227161bb31e2f4e50aa398a72291fbbdd11a4068 Mon Sep 17 00:00:00 2001 From: Philipp Erhardt Date: Sun, 10 Nov 2019 11:22:56 +0100 Subject: [PATCH 002/512] Add cookie support for all rippers --- .../java/com/rarchives/ripme/utils/Http.java | 45 ++++++++++++++++++- .../com/rarchives/ripme/utils/RipUtils.java | 2 +- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index 885a194d..1776463a 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -1,15 +1,19 @@ package com.rarchives.ripme.utils; import java.io.IOException; +import java.net.MalformedURLException; import java.net.URL; import java.util.HashMap; import java.util.Map; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.json.JSONObject; import org.jsoup.Connection; import org.jsoup.Connection.Method; import org.jsoup.Connection.Response; +import org.jsoup.helper.StringUtil; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -22,8 +26,8 @@ import com.rarchives.ripme.ripper.AbstractRipper; */ public class Http { - private static final int TIMEOUT = Utils.getConfigInteger("page.timeout", 5 * 1000); - private static final Logger logger = Logger.getLogger(Http.class); + private static final int TIMEOUT = Utils.getConfigInteger("page.timeout", 5 * 1000); + private static final Logger logger = Logger.getLogger(Http.class); private int retries; private String url; @@ -53,6 +57,43 @@ public class Http { connection.method(Method.GET); connection.timeout(TIMEOUT); connection.maxBodySize(0); + + // Extract cookies from config entry: + // Example config entry: + // cookies.reddit.com = reddit_session=; other_cookie= + connection.cookies(cookiesForURL(this.url)); + } + + private Map cookiesForURL(String u) { + Map cookiesParsed = new HashMap<>(); + + try { + URL parsed = new URL(this.url); + String cookieStr = ""; + + String[] parts = parsed.getHost().split("\\."); + + // if url is www.reddit.com, we should also use cookies from reddit.com; + // this rule is applied for all subdomains (for all rippers); e.g. also + // old.reddit.com, new.reddit.com + while (parts.length > 1) { + // Try to get cookies for this host from config + cookieStr = Utils.getConfigString("cookies." + String.join(".", parts), ""); + if (cookieStr != "") { + // we found something, start parsing + break; + } + parts = (String[]) ArrayUtils.remove(parts, 0); + } + + if (cookieStr != "") { + cookiesParsed = RipUtils.getCookiesFromString(cookieStr.trim()); + } + } catch (MalformedURLException e) { + logger.warn("Parsing url while getting cookies" + url, e); + } + + return cookiesParsed; } // Setters diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 5dea166b..03a480cf 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -301,7 +301,7 @@ public class RipUtils { Map cookies = new HashMap<>(); for (String pair : line.split(";")) { String[] kv = pair.split("="); - cookies.put(kv[0], kv[1]); + cookies.put(kv[0].trim(), kv[1]); } return cookies; } From 7fe3ce059b53cc2a41712484287e2ff79bf34a0f Mon Sep 17 00:00:00 2001 From: Philipp Erhardt Date: Mon, 18 Nov 2019 16:56:55 +0100 Subject: [PATCH 003/512] Use the passed url in cookiesForURL function While this doesn't make any difference, not using the argument is kind of bad. One could also remove the argument and use `this.url` directly. --- src/main/java/com/rarchives/ripme/utils/Http.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index 1776463a..0eabaea1 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -68,7 +68,7 @@ public class Http { Map cookiesParsed = new HashMap<>(); try { - URL parsed = new URL(this.url); + URL parsed = new URL(u); String cookieStr = ""; String[] parts = parsed.getHost().split("\\."); From 152d6f635fd4764e56a4dee34ca7fc2a9bc290b4 Mon Sep 17 00:00:00 2001 From: Philipp Erhardt Date: Thu, 21 Nov 2019 20:02:02 +0100 Subject: [PATCH 004/512] Warn users about possibly fixable permission error --- .../java/com/rarchives/ripme/utils/Http.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index 0eabaea1..5ce9c48b 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -15,6 +15,7 @@ import org.jsoup.Connection.Method; import org.jsoup.Connection.Response; import org.jsoup.helper.StringUtil; import org.jsoup.Jsoup; +import org.jsoup.HttpStatusException; import org.jsoup.nodes.Document; import com.rarchives.ripme.ripper.AbstractRipper; @@ -90,7 +91,7 @@ public class Http { cookiesParsed = RipUtils.getCookiesFromString(cookieStr.trim()); } } catch (MalformedURLException e) { - logger.warn("Parsing url while getting cookies" + url, e); + logger.warn("Parsing url " + u + " while getting cookies", e); } return cookiesParsed; @@ -171,6 +172,20 @@ public class Http { response = connection.execute(); return response; } catch (IOException e) { + // Warn users about possibly fixable permission error + if (e instanceof org.jsoup.HttpStatusException) { + HttpStatusException ex = (HttpStatusException)e; + + // These status codes might indicate missing cookies + // 401 Unauthorized + // 403 Forbidden + + int status = ex.getStatusCode(); + if (status == 401 || status == 403) { + throw new IOException("Failed to load " + url + ": Status Code " + Integer.toString(status) + ". You might be able to circumvent this error by setting cookies for this domain" , e); + } + } + logger.warn("Error while loading " + url, e); lastException = e; } From 6c330c6932dd66b7ab25d8f6e36a207e5d827439 Mon Sep 17 00:00:00 2001 From: Philipp Erhardt Date: Sun, 1 Dec 2019 11:33:08 +0100 Subject: [PATCH 005/512] Add a logging statement if cookies were added to a request --- src/main/java/com/rarchives/ripme/utils/Http.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index 5ce9c48b..d2796b53 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -68,6 +68,7 @@ public class Http { private Map cookiesForURL(String u) { Map cookiesParsed = new HashMap<>(); + String cookieDomain = ""; try { URL parsed = new URL(u); String cookieStr = ""; @@ -78,9 +79,11 @@ public class Http { // this rule is applied for all subdomains (for all rippers); e.g. also // old.reddit.com, new.reddit.com while (parts.length > 1) { + String domain = String.join(".", parts); // Try to get cookies for this host from config - cookieStr = Utils.getConfigString("cookies." + String.join(".", parts), ""); + cookieStr = Utils.getConfigString("cookies." + domain, ""); if (cookieStr != "") { + cookieDomain = domain; // we found something, start parsing break; } @@ -94,6 +97,10 @@ public class Http { logger.warn("Parsing url " + u + " while getting cookies", e); } + if (cookiesParsed.size() > 0) { + logger.info("Cookies for " + cookieDomain + " have been added to this request"); + } + return cookiesParsed; } From 74bfac0e3efc331588b6eebf8ce78ac0733ab066 Mon Sep 17 00:00:00 2001 From: Philipp Erhardt Date: Sun, 1 Dec 2019 11:45:45 +0100 Subject: [PATCH 006/512] String comparisons: don't use "reference equality" See https://stackoverflow.com/q/513832 for more details --- src/main/java/com/rarchives/ripme/utils/Http.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index d2796b53..1b85005c 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -82,7 +82,7 @@ public class Http { String domain = String.join(".", parts); // Try to get cookies for this host from config cookieStr = Utils.getConfigString("cookies." + domain, ""); - if (cookieStr != "") { + if (cookieStr.equals("")) { cookieDomain = domain; // we found something, start parsing break; @@ -90,7 +90,7 @@ public class Http { parts = (String[]) ArrayUtils.remove(parts, 0); } - if (cookieStr != "") { + if (!cookieStr.equals("")) { cookiesParsed = RipUtils.getCookiesFromString(cookieStr.trim()); } } catch (MalformedURLException e) { From b1e9adfbf36f138fb721b2790b56bff3a0515e40 Mon Sep 17 00:00:00 2001 From: Tushar Date: Tue, 10 Dec 2019 01:04:44 +0530 Subject: [PATCH 007/512] Fixed VkRipper not ripping images and bumped json library. --- pom.xml | 2 +- .../ripme/ripper/rippers/VkRipper.java | 258 ++++++++++++------ .../tst/ripper/rippers/VkRipperTest.java | 28 +- 3 files changed, 201 insertions(+), 87 deletions(-) diff --git a/pom.xml b/pom.xml index 9f86678f..2d6f9d62 100644 --- a/pom.xml +++ b/pom.xml @@ -46,7 +46,7 @@ org.json json - 20140107 + 20190722 commons-configuration diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java index 99310dc4..b364a5ae 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java @@ -6,10 +6,12 @@ import java.net.URL; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; - +import org.apache.commons.lang.StringEscapeUtils; import com.rarchives.ripme.ripper.AbstractJSONRipper; import org.json.JSONArray; import org.json.JSONObject; +import org.jsoup.Connection.Method; +import org.jsoup.Connection.Response; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -26,6 +28,7 @@ public class VkRipper extends AbstractJSONRipper { private RipType RIP_TYPE; private String oid; + private int offset = 0; public VkRipper(URL url) throws IOException { super(url); @@ -59,68 +62,18 @@ public class VkRipper extends AbstractJSONRipper { String[] jsonStrings = doc.toString().split(""); return new JSONObject(jsonStrings[jsonStrings.length - 1]); } else { - Map photoIDsToURLs = new HashMap<>(); - int offset = 0; - while (true) { - LOGGER.info(" Retrieving " + this.url); - Map postData = new HashMap<>(); - postData.put("al", "1"); - postData.put("offset", Integer.toString(offset)); - postData.put("part", "1"); - Document doc = Http.url(this.url) - .referrer(this.url) - .ignoreContentType() - .data(postData) - .post(); - - String body = doc.toString(); - if (!body.contains(" elements = doc.select("a"); - Set photoIDsToGet = new HashSet<>(); - for (Element a : elements) { - if (!a.attr("onclick").contains("showPhoto('")) { - LOGGER.error("a: " + a); - continue; - } - String photoID = a.attr("onclick"); - photoID = photoID.substring(photoID.indexOf("showPhoto('") + "showPhoto('".length()); - photoID = photoID.substring(0, photoID.indexOf("'")); - if (!photoIDsToGet.contains(photoID)) { - photoIDsToGet.add(photoID); - } - } - for (String photoID : photoIDsToGet) { - if (!photoIDsToURLs.containsKey(photoID)) { - try { - photoIDsToURLs.putAll(getPhotoIDsToURLs(photoID)); - } catch (IOException e) { - LOGGER.error("Exception while retrieving photo id " + photoID, e); - continue; - } - } - if (!photoIDsToURLs.containsKey(photoID)) { - LOGGER.error("Could not find URL for photo ID: " + photoID); - continue; - } - if (isStopped() || isThisATest()) { - break; - } - } - - if (elements.size() < 40 || isStopped() || isThisATest()) { - break; - } - offset += elements.size(); - } - // Slight hack to make this into effectively a JSON ripper - return new JSONObject(photoIDsToURLs); + return getPage(); } } + @Override + protected JSONObject getNextPage(JSONObject doc) throws IOException { + if (isStopped() || isThisATest()) { + return null; + } + return getPage(); + } + @Override protected List getURLsFromJSON(JSONObject page) { List pageURLs = new ArrayList<>(); @@ -142,9 +95,9 @@ public class VkRipper extends AbstractJSONRipper { pageURLs.add(videoURL); } } else { - Iterator keys = page.keys(); + Iterator keys = page.keys(); while (keys.hasNext()) { - pageURLs.add(page.getString((String) keys.next())); + pageURLs.add(page.getString(keys.next())); } } return pageURLs; @@ -197,6 +150,7 @@ public class VkRipper extends AbstractJSONRipper { else { RIP_TYPE = RipType.IMAGE; } + super.rip(); } private Map getPhotoIDsToURLs(String photoID) throws IOException { @@ -208,40 +162,182 @@ public class VkRipper extends AbstractJSONRipper { postData.put("al", "1"); postData.put("module", "photos"); postData.put("photo", photoID); - Document doc = Jsoup - .connect("https://vk.com/al_photos.php") + Response res = Jsoup.connect("https://vk.com/al_photos.php") .header("Referer", this.url.toExternalForm()) + .header("Accept", "*/*") + .header("Accept-Language", "en-US,en;q=0.5") + .header("Content-Type", "application/x-www-form-urlencoded") + .header("X-Requested-With", "XMLHttpRequest") .ignoreContentType(true) .userAgent(USER_AGENT) .timeout(5000) .data(postData) - .post(); - String jsonString = doc.toString(); - jsonString = jsonString.substring(jsonString.indexOf("") + "".length()); - jsonString = jsonString.substring(0, jsonString.indexOf("")); - JSONArray json = new JSONArray(jsonString); - for (int i = 0; i < json.length(); i++) { - JSONObject jsonImage = json.getJSONObject(i); - for (String key : new String[] {"z_src", "y_src", "x_src"}) { - if (!jsonImage.has(key)) { - continue; - } - photoIDsToURLs.put(jsonImage.getString("id"), jsonImage.getString(key)); - break; - } + .method(Method.POST) + .execute(); + String jsonString = res.body(); + JSONObject json = new JSONObject(jsonString); + JSONObject photoObject = findJSONObjectContainingPhotoId(photoID, json); + String bestSourceUrl = getBestSourceUrl(photoObject); + + if (bestSourceUrl != null) { + photoIDsToURLs.put(photoID, bestSourceUrl); + } else { + LOGGER.error("Could not find image source for " + photoID); } + return photoIDsToURLs; } @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://(www\\.)?vk\\.com/(photos|album|videos)-?([a-zA-Z0-9_]+).*$"); + Pattern p = Pattern.compile("^https?:\\/\\/(?:www\\.)?vk\\.com\\/((?:photos|album|videos)-?(?:[a-zA-Z0-9_]+).*$)"); Matcher m = p.matcher(url.toExternalForm()); if (!m.matches()) { throw new MalformedURLException("Expected format: http://vk.com/album#### or vk.com/photos####"); } - int count = m.groupCount(); - return m.group(count - 1) + m.group(count); + return m.group(1); } + + /** + * Finds the nested JSON object with entry "id": "photoID" recursively. + * @param photoID The photoId string to be found with "id" as the key. + * @param json Object of type JSONObject or JSONArray. + * @return JSONObject with id as the photoID or null. + */ + public JSONObject findJSONObjectContainingPhotoId(String photoID, Object json) { + // Termination condition + if (json instanceof JSONObject && ((JSONObject) json).has("id") + && ((JSONObject) json).optString("id").equals(photoID)) { + return ((JSONObject) json); + } + + if (json instanceof JSONObject) { + // Iterate through every key:value pair in the json. + Iterator iterator = ((JSONObject) json).keys(); + while (iterator.hasNext()) { + Object o = ((JSONObject) json).get(iterator.next()); + JSONObject responseJson = findJSONObjectContainingPhotoId(photoID, o); + if (responseJson != null) { + return responseJson; + } + } + + } + + if (json instanceof JSONArray) { + // Iterate through every array value in the json + for (Object o : (JSONArray) json) { + if (o instanceof JSONObject || o instanceof JSONArray) { + JSONObject responseJson = findJSONObjectContainingPhotoId(photoID, o); + if (responseJson != null) { + return responseJson; + } + } + } + } + + return null; + } + + /** + * Find the best source url( with highest resolution). + * @param json JSONObject containing src urls. + * @return Url string for the image src or null. + */ + public String getBestSourceUrl(JSONObject json) { + String bestSourceKey = null; + int bestSourceResolution = 0; + Iterator iterator = json.keys(); + + while (iterator.hasNext()) { + String key = iterator.next(); + Object o = json.get(key); + // JSON contains source urls in the below format. Check VkRipperTest.java for sample json. + // {..., + // "x_src":"src-url", + // "x_": ["incomplete-url", width, height], + // ...} + if (o instanceof JSONArray && ((JSONArray) o).length() == 3 + && !((JSONArray) o).optString(0).equals("") && ((JSONArray) o).optInt(1) != 0 + && ((JSONArray) o).optInt(2) != 0 && json.has(key + "src")) { + if (((JSONArray) o).optInt(1) * ((JSONArray) o).optInt(2) >= bestSourceResolution) { + bestSourceResolution = ((JSONArray) o).optInt(1) * ((JSONArray) o).optInt(2); + bestSourceKey = key; + } + } + } + + // In case no suitable source has been found, we fall back to the older way. + if(bestSourceKey == null) { + for (String key : new String[] {"z_src", "y_src", "x_src", "w_src"}) { + if(!json.has(key)) { + continue; + } + return json.getString(key); + } + }else { + return json.getString(bestSourceKey + "src"); + } + + return null; + } + + /** + * Common function to get the next page( containing next batch of images). + * @return JSONObject containing entries of "imgId": "src" + * @throws IOException + */ + private JSONObject getPage() throws IOException { + Map photoIDsToURLs = new HashMap<>(); + Map postData = new HashMap<>(); + + LOGGER.info("Retrieving " + this.url + " from offset " + offset); + postData.put("al", "1"); + postData.put("offset", Integer.toString(offset)); + postData.put("part", "1"); + Document doc = + Http.url(this.url).referrer(this.url).ignoreContentType().data(postData).post(); + String body = doc.toString(); + if (!body.contains(" elements = doc.select("a"); + Set photoIDsToGet = new HashSet<>(); + for (Element a : elements) { + if (!a.attr("onclick").contains("showPhoto('")) { + continue; + } + String photoID = a.attr("onclick"); + photoID = photoID.substring(photoID.indexOf("showPhoto('") + "showPhoto('".length()); + photoID = photoID.substring(0, photoID.indexOf("'")); + if (!photoIDsToGet.contains(photoID)) { + photoIDsToGet.add(photoID); + } + } + for (String photoID : photoIDsToGet) { + if (!photoIDsToURLs.containsKey(photoID)) { + try { + photoIDsToURLs.putAll(getPhotoIDsToURLs(photoID)); + } catch (IOException e) { + LOGGER.error("Exception while retrieving photo id " + photoID, e); + continue; + } + } + if (!photoIDsToURLs.containsKey(photoID)) { + LOGGER.error("Could not find URL for photo ID: " + photoID); + continue; + } + if (isStopped() || isThisATest()) { + break; + } + } + + offset += elements.size(); + // Slight hack to make this into effectively a JSON ripper + return new JSONObject(photoIDsToURLs); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java index 22ccb641..327698bd 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.VkRipper; +import org.json.JSONObject; import org.junit.jupiter.api.Test; public class VkRipperTest extends RippersTest { @@ -17,11 +18,6 @@ public class VkRipperTest extends RippersTest { // EXAMPLE: https://vk.com/album45506334_101886701 (a single album - custom) @Test public void testVkAlbumHttpRip() throws IOException { - VkRipper ripper = new VkRipper(new URL("http://vk.com/album45506334_0")); - testRipper(ripper); - } - @Test - public void testVkAlbumHttpsRip() throws IOException { VkRipper ripper = new VkRipper(new URL("https://vk.com/album45506334_0")); testRipper(ripper); } @@ -30,4 +26,26 @@ public class VkRipperTest extends RippersTest { VkRipper ripper = new VkRipper(new URL("https://vk.com/photos45506334")); testRipper(ripper); } + + @Test + public void testFindJSONObjectContainingPhotoID() throws IOException { + VkRipper ripper = new VkRipper(new URL("http://vk.com/album45506334_0")); + String json = + "{\"payload\":[0,[\"album-45984105_268691406\",18,14,[{\"id\":\"-45984105_457345201\",\"base\":\"https://sun9-37.userapi.com/\",\"tagged\":[],\"likes\":0,\"shares\":0,\"o_src\":\"https://sun9-65.userapi.com/c857520/v857520962/10e24c/DPxygc3XW5E.jpg\",\"o_\":[\"https://sun9-65.userapi.com/c857520/v857520962/10e24c/DPxygc3XW5E\",130,98],\"z_src\":\"https://sun9-41.userapi.com/c857520/v857520962/10e24a/EsDDQA36qKI.jpg\",\"z_\":[\"https://sun9-41.userapi.com/c857520/v857520962/10e24a/EsDDQA36qKI\",1280,960],\"w_src\":\"https://sun9-60.userapi.com/c857520/v857520962/10e24b/6ETsA15rAdU.jpg\",\"w_\":[\"https://sun9-60.userapi.com/c857520/v857520962/10e24b/6ETsA15rAdU\",1405,1054]}]]],\"langVersion\":\"4298\"}"; + String responseJson = + "{\"id\":\"-45984105_457345201\",\"base\":\"https://sun9-37.userapi.com/\",\"tagged\":[],\"likes\":0,\"shares\":0,\"o_src\":\"https://sun9-65.userapi.com/c857520/v857520962/10e24c/DPxygc3XW5E.jpg\",\"o_\":[\"https://sun9-65.userapi.com/c857520/v857520962/10e24c/DPxygc3XW5E\",130,98],\"z_src\":\"https://sun9-41.userapi.com/c857520/v857520962/10e24a/EsDDQA36qKI.jpg\",\"z_\":[\"https://sun9-41.userapi.com/c857520/v857520962/10e24a/EsDDQA36qKI\",1280,960],\"w_src\":\"https://sun9-60.userapi.com/c857520/v857520962/10e24b/6ETsA15rAdU.jpg\",\"w_\":[\"https://sun9-60.userapi.com/c857520/v857520962/10e24b/6ETsA15rAdU\",1405,1054]}"; + + assertTrue( + ripper.findJSONObjectContainingPhotoId("-45984105_457345201", new JSONObject(json)) + .similar(new JSONObject(responseJson))); + } + + @Test + public void testGetBestSourceUrl() throws IOException { + VkRipper ripper = new VkRipper(new URL("http://vk.com/album45506334_0")); + String json = + "{\"id\":\"-45984105_457345201\",\"base\":\"https://sun9-37.userapi.com/\",\"commcount\":0,\"date\":\"3 Dec at 1:14 am\",\"tagged\":[],\"attached_tags\":{\"max_tags_per_object\":5},\"o_src\":\"https://sun9-65.userapi.com/c857520/v857520962/10e24c/DPxygc3XW5E.jpg\",\"o_\":[\"https://sun9-65.userapi.com/c857520/v857520962/10e24c/DPxygc3XW5E\",130,98],\"y_src\":\"https://sun9-9.userapi.com/c857520/v857520962/10e249/dUDeuY10s0A.jpg\",\"y_\":[\"https://sun9-9.userapi.com/c857520/v857520962/10e249/dUDeuY10s0A\",807,605],\"z_src\":\"https://sun9-41.userapi.com/c857520/v857520962/10e24a/EsDDQA36qKI.jpg\",\"z_\":[\"https://sun9-41.userapi.com/c857520/v857520962/10e24a/EsDDQA36qKI\",1280,960]}"; + assertEquals("https://sun9-41.userapi.com/c857520/v857520962/10e24a/EsDDQA36qKI.jpg", + ripper.getBestSourceUrl(new JSONObject(json))); + } } From 273f698ad4a259b5518f45ef3f3bf43370139e6a Mon Sep 17 00:00:00 2001 From: Tushar Date: Sun, 29 Dec 2019 16:25:14 +0530 Subject: [PATCH 008/512] Fixed luscious ripper. --- .../ripme/ripper/rippers/LusciousRipper.java | 38 ++++++++++++++++++- .../ripper/rippers/LusciousRipperTest.java | 6 ++- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java index e56f8dbc..7eabfc6f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java @@ -19,7 +19,7 @@ import com.rarchives.ripme.utils.Http; public class LusciousRipper extends AbstractHTMLRipper { private static final int RETRY_COUNT = 5; // Keeping it high for read timeout exception. - private Pattern p = Pattern.compile("^https?://(?:members.)?luscious\\.net/albums/([-_.0-9a-zA-Z]+).*$"); + private static final Pattern P = Pattern.compile("^https?:\\/\\/(?:members\\.|old\\.|www\\.)?luscious.net\\/albums\\/([-_.0-9a-zA-Z]+)\\/?"); private DownloadThreadPool lusciousThreadPool = new DownloadThreadPool("lusciousThreadPool"); public LusciousRipper(URL url) throws IOException { @@ -69,7 +69,7 @@ public class LusciousRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Matcher m = p.matcher(url.toExternalForm()); + Matcher m = P.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); } @@ -87,6 +87,40 @@ public class LusciousRipper extends AbstractHTMLRipper { return lusciousThreadPool; } + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + // Sanitizes the url removing GET parameters and convert to old api url. + // "https://old.luscious.net/albums/albumname" + try { + Matcher m = P.matcher(url.toString()); + if (m.matches()) { + String sanitizedUrl = m.group(); + sanitizedUrl = sanitizedUrl.replaceFirst( + "^https?:\\/\\/(?:members\\.|old\\.|www\\.)?luscious.net", + "https://old.luscious.net"); + return new URL(sanitizedUrl); + } + + throw new Exception("ERROR: Unable to sanitize url."); + } catch (Exception e) { + LOGGER.info("Error sanitizing the url."); + LOGGER.error(e); + return super.sanitizeURL(url); + } + } + + @Override + public String normalizeUrl(String url) { + try { + return url.toString().replaceFirst( + "^https?:\\/\\/(?:members\\.|old\\.)?luscious.net", "https://www.luscious.net"); + } catch (Exception e) { + LOGGER.info("Error normalizing the url."); + LOGGER.error(e); + return super.normalizeUrl(url); + } + } + public class LusciousDownloadThread extends Thread { private URL url; private int index; diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java index 6362dcc6..c6febd32 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java @@ -8,7 +8,7 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class LusciousRipperTest extends RippersTest { - @Test @Disabled("Flaky in the CI") + @Test public void testPahealRipper() throws IOException { // a photo set LusciousRipper ripper = new LusciousRipper( @@ -16,12 +16,14 @@ public class LusciousRipperTest extends RippersTest { testRipper(ripper); } + @Test public void testGetGID() throws IOException { URL url = new URL("https://luscious.net/albums/h-na-alice-wa-suki-desu-ka-do-you-like-alice-when_321609/"); LusciousRipper ripper = new LusciousRipper(url); assertEquals("h-na-alice-wa-suki-desu-ka-do-you-like-alice-when_321609", ripper.getGID(url)); } - @Test @Disabled("Flaky in the CI") + + @Test public void testGetNextPage() throws IOException { URL multiPageAlbumUrl = new URL("https://luscious.net/albums/women-of-color_58/"); LusciousRipper multiPageRipper = new LusciousRipper(multiPageAlbumUrl); From c520f51916c8163c0c92719d0e45f8d28130e3d3 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 17 Jan 2020 15:18:29 -0500 Subject: [PATCH 009/512] Added kingcomix ripper --- .../ripme/ripper/rippers/KingcomixRipper.java | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/KingcomixRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/KingcomixRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/KingcomixRipper.java new file mode 100644 index 00000000..4876237e --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/KingcomixRipper.java @@ -0,0 +1,64 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class KingcomixRipper extends AbstractHTMLRipper { + + public KingcomixRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "kingcomix"; + } + + @Override + public String getDomain() { + return "kingcomix.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https://kingcomix.com/([a-zA-Z1-9_-]*)/?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected kingcomix URL format: " + + "kingcomix.com/COMIX - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + for (Element el : doc.select("div.entry-content > p > img")) { + result.add(el.attr("src")); + } + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} From 6b806c331c477281f43bf3a8f193840dc46fd4eb Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 17 Jan 2020 15:25:12 -0500 Subject: [PATCH 010/512] Added kingcomix.com ripper and test --- .../ripper/rippers/KingcomixRipperTest.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/KingcomixRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/KingcomixRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/KingcomixRipperTest.java new file mode 100644 index 00000000..ebe23e4c --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/KingcomixRipperTest.java @@ -0,0 +1,25 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.KingcomixRipper; + +import org.junit.jupiter.api.Test; + +public class KingcomixRipperTest extends RippersTest { + + @Test + public void testRip() throws IOException { + KingcomixRipper ripper = new KingcomixRipper(new URL("https://kingcomix.com/aunt-cumming-tracy-scops/")); + testRipper(ripper); + } + + @Test + public void testGetGID() throws IOException { + URL url = new URL("https://kingcomix.com/aunt-cumming-tracy-scops/"); + KingcomixRipper ripper = new KingcomixRipper(url); + assertEquals("aunt-cumming-tracy-scops", ripper.getGID(url)); + } + +} From 4248a0fd231dde73e1d3b40d24949384d244805a Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 20 Jan 2020 03:20:11 -0500 Subject: [PATCH 011/512] 1.7.91: Fixed luscious ripper. Fixed VK ripper; Added Kingcomix ripper --- pom.xml | 2 +- ripme.json | 5 +++-- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 2d6f9d62..408d0fd4 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.rarchives.ripme ripme jar - 1.7.90 + 1.7.91 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 5d9fa969..6e0d56a8 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "currentHash": "a2fdb180da195c617cff933fc400d16010d049580188a8eae7eb89e11bd0d4ef", + "currentHash": "fb1e97064f255315da31b794897a9aacf2f958aad89202677deb6cdcf565ec77", "changeList": [ + "1.7.91: Fixed luscious ripper. Fixed VK ripper; Added Kingcomix ripper", "1.7.90: Added FitnakedgirlsRipper; Fixed VK Album Ripper; Fixed Myreadingmanga Ripper; Fixed windows max file name; Fixed Pornhub Video Ripper; Fixed Motherless Ripper; Fixed Instagram Ripper", "1.7.89: Improved twitter ripper; Fixed xhamster image ripper; Fixed allporncomic ripper; Added Ripper for folio.ink", "1.7.88: Added ripper for Myreadingmanga.info; Added Mastodon rippers; Fix queue count update when queue is 0; Added ripper for listal; Now downloads best video when ripping twitter", @@ -262,5 +263,5 @@ "1.0.3: Added VK.com ripper", "1.0.1: Added auto-update functionality" ], - "latestVersion": "1.7.90" + "latestVersion": "1.7.91" } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 42f2ad8c..86760f1e 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -23,7 +23,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.90"; + private static final String DEFAULT_VERSION = "1.7.91"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static String mainFileName; From 3d29d4cfcfd10a0dc3bec925f517c83f283c3f8e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 21 Jan 2020 02:07:25 -0500 Subject: [PATCH 012/512] Added read-comic.com ripper --- .../ripme/ripper/rippers/ReadcomicRipper.java | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/ReadcomicRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ReadcomicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ReadcomicRipper.java new file mode 100644 index 00000000..55b3559a --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ReadcomicRipper.java @@ -0,0 +1,55 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class ReadcomicRipper extends ViewcomicRipper { + + public ReadcomicRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "read-comic"; + } + + @Override + public String getDomain() { + return "read-comic.com"; + } + + + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://read-comic.com/([a-zA-Z1-9_-]*)/?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected view-comic URL format: " + + "read-comic.com/COMIC_NAME - got " + url + " instead"); + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList(); + for (Element el : doc.select("div.pinbin-copy > a > img")) { + result.add(el.attr("src")); + } + return result; + } + +} From 5555e2017b463265af5d81f89425880fa12bd882 Mon Sep 17 00:00:00 2001 From: 0x1f595 <0x1f595@users.noreply.github.com> Date: Tue, 21 Jan 2020 18:29:17 -0700 Subject: [PATCH 013/512] Fix Pawoo ripper pagination The generic Mastodon ripper mostly worked as a base for this ripper, but Pawoo uses a custom web UI that changes how navigation links work, so I've updated it to use the correct selector. --- .../ripme/ripper/rippers/PawooRipper.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PawooRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PawooRipper.java index 8f5c8c37..31817c84 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PawooRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PawooRipper.java @@ -3,6 +3,11 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.URL; +import com.rarchives.ripme.utils.Http; + +import org.jsoup.nodes.Document; +import org.jsoup.select.Elements; + public class PawooRipper extends MastodonRipper { public PawooRipper(URL url) throws IOException { super(url); @@ -17,4 +22,17 @@ public class PawooRipper extends MastodonRipper { public String getDomain() { return "pawoo.net"; } + + + @Override + // Pawoo uses a custom theme that has different navigation links + public Document getNextPage(Document doc) throws IOException { + Elements hrefs = doc.select(".pagination a[rel=\"next\"]"); + if (hrefs.isEmpty()) { + throw new IOException("No more pages"); + } + String nextUrl = hrefs.last().attr("href"); + sleep(500); + return Http.url(nextUrl).get(); + } } From 8b212c66268bb944b611320a14166a1c22f9e1e0 Mon Sep 17 00:00:00 2001 From: YanJun Sun <34938411+YanJunSunYJS@users.noreply.github.com> Date: Tue, 3 Mar 2020 10:29:15 +0800 Subject: [PATCH 014/512] Add ChineseSimplified language file --- src/LabelsBundle_zh_CN.properties | 75 +++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 src/LabelsBundle_zh_CN.properties diff --git a/src/LabelsBundle_zh_CN.properties b/src/LabelsBundle_zh_CN.properties new file mode 100644 index 00000000..994efcf8 --- /dev/null +++ b/src/LabelsBundle_zh_CN.properties @@ -0,0 +1,75 @@ +Log = 日志 +History = 历史 +created = 创建时间 +modified = 修改时间 +queue = 队列 +Configuration = 配置 +open = 打开 + +# Keys for the Configuration menu +current.version = 当前版本 +check.for.updates = 检查更新 +auto.update = 自动更新？ +max.download.threads = 最大下载线程数： +timeout.mill = 超时（毫秒）： +retry.download.count = 重试下载次数 +overwrite.existing.files = 覆盖现有文件？ +sound.when.rip.completes = 抓取完成时播放声音 +preserve.order = 保持顺序 +save.logs = 保存日志 +notification.when.rip.starts = 通知抓取开始 +save.urls.only = 仅保存 URL +save.album.titles = 保存专辑标题 +autorip.from.clipboard = 监视剪贴板上的 URL +save.descriptions = 保存描述 +prefer.mp4.over.gif = 首选 MP4 而非 GIF +restore.window.position = 恢复窗口位置 +remember.url.history = 记住 URL 历史 +loading.history.from = 加载历史从 + +# Queue keys +queue.remove.all = 移除全部 +queue.validation = 您确定要移除队列内的全部项目？ +queue.remove.selected = 移除所选项目 + +# History +re-rip.checked = 重新抓取选中的项目 +remove = 移除 +clear = 清除 +history.check.all = 选中全部 +history.check.none = 取消选中全部 +history.check.selected = 选中所选项目 +history.uncheck.selected = 取消选中所选项目 +history.load.failed.warning = RipMe 加载位于 historyFile.getAbsolutePath() 的历史文件失败\n\n错误：%s\n\n关闭 RipMe 会自动覆盖此文件的内容，\n请在关闭 RipMe 前备份它！ +history.load.none = 无可重新抓取的历史条目。请先抓取一些专辑 +history.load.none.checked = 未 '选中' 任何历史条目，请通过选中所需 URL 前面的复选框或URL 的右键菜单以选中所需条目 + +# TrayIcon +tray.show = 显示 +tray.hide = 隐藏 +tray.autorip = 监视剪贴板上的 URL +tray.exit = 退出 + +# Misc UI keys +loading.history.from.configuration = 从配置加载历史 +interrupted.while.waiting.to.rip.next.album = 等候抓取下一专辑期间发生中断 +inactive = 非活动 +download.url.list = 下载 URL 列表 +select.save.dir = 选择保存目录 + +# Keys for the logs generated by DownloadFileThread +nonretriable.status.code = 非可重试状态代码 +retriable.status.code = 可重试状态代码 +server.doesnt.support.resuming.downloads = 服务器不支持继续下载（续传） +# A "magic number" can also be called a file signature +was.unable.to.get.content.type.using.magic.number = 不能使用幻数获取内容类型 +magic.number.was = 幻数为 +deleting.existing.file = 删除现有文件 +request.properties = 请求属性 +download.interrupted = 下载中断 +exceeded.maximum.retries = 超过最大重试次数 +http.status.exception = HTTP 状态意外 +exception.while.downloading.file = 下载文件时发生意外 +failed.to.download = 下载失败 +skipping = 跳过 +file.already.exists = 文件已存在 \ No newline at end of file From 1c2bfe2ea57d513b2de9f44b9fbd745ff1918678 Mon Sep 17 00:00:00 2001 From: Felix Friebe Date: Thu, 5 Mar 2020 19:50:58 +0100 Subject: [PATCH 015/512] Due to HTTP compression response was not readable --> deactivated in HTTP Headers --- .../rarchives/ripme/ripper/rippers/ArtStationRipper.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java index f61f48f8..66455861 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java @@ -60,7 +60,7 @@ public class ArtStationRipper extends AbstractJSONRipper { // URL points to user portfolio, use user's full name as GID String userInfoURL = "https://www.artstation.com/users/" + albumURL.getID() + "/quick.json"; try { - // groupData = Http.url(userInfoURL).getJSON(); +// groupData = Http.url(userInfoURL).getJSON(); groupData = getJson(userInfoURL); } catch (IOException e) { throw new MalformedURLException("Couldn't load JSON from " + userInfoURL); @@ -254,7 +254,7 @@ public class ArtStationRipper extends AbstractJSONRipper { con.userAgent("Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0"); con.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); con.header("Accept-Language", "en-US,en;q=0.5"); - con.header("Accept-Encoding", "gzip, deflate, br"); +// con.header("Accept-Encoding", "gzip, deflate, br"); con.header("Upgrade-Insecure-Requests", "1"); Response res = con.execute(); int status = res.statusCode(); @@ -309,7 +309,7 @@ public class ArtStationRipper extends AbstractJSONRipper { "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11"); con.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); con.header("Accept-Language", "en-US,en;q=0.5"); - con.header("Accept-Encoding", "gzip, deflate, br"); +// con.header("Accept-Encoding", "gzip, deflate, br"); con.header("Upgrade-Insecure-Requests", "1"); Response res = con.execute(); int status = res.statusCode(); From 9b31254df4c03bf1e8ce09aff2556683763424d2 Mon Sep 17 00:00:00 2001 From: Felix Friebe Date: Sat, 7 Mar 2020 02:26:57 +0100 Subject: [PATCH 016/512] URL to galleries changed --- .../com/rarchives/ripme/ripper/rippers/VscoRipper.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java index 5b23b8bd..dca72913 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java @@ -24,7 +24,7 @@ public class VscoRipper extends AbstractHTMLRipper { private static final String DOMAIN = "vsco.co", - HOST = "vsco"; + HOST = "vsco"; public VscoRipper(URL url) throws IOException{ super(url); @@ -101,7 +101,7 @@ public class VscoRipper extends AbstractHTMLRipper { private String getUserTkn(String username) { String userinfoPage = "https://vsco.co/content/Static/userinfo"; - String referer = "https://vsco.co/" + username + "/images/1"; + String referer = "https://vsco.co/" + username + "/gallery"; Map cookies = new HashMap<>(); cookies.put("vs_anonymous_id", UUID.randomUUID().toString()); try { @@ -116,7 +116,7 @@ public class VscoRipper extends AbstractHTMLRipper { } private String getUserName() { - Pattern p = Pattern.compile("^https?://vsco.co/([a-zA-Z0-9-]+)/images/[0-9]+"); + Pattern p = Pattern.compile("^https?://vsco.co/([a-zA-Z0-9-]+)/gallery(/)?"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { @@ -200,7 +200,7 @@ public class VscoRipper extends AbstractHTMLRipper { } //Member profile (Usernames should all be different, so this should work. - p = Pattern.compile("^https?://vsco.co/([a-zA-Z0-9-]+)/images/[0-9]+"); + p = Pattern.compile("^https?://vsco.co/([a-zA-Z0-9-]+)/gallery(/)?"); m = p.matcher(url.toExternalForm()); if (m.matches()){ From 677e90ba88fc40666cb4e5c470e2f8f12acd9ac7 Mon Sep 17 00:00:00 2001 From: Felix Friebe Date: Sat, 7 Mar 2020 02:32:55 +0100 Subject: [PATCH 017/512] URL to galleries changed --- .../java/com/rarchives/ripme/ripper/rippers/VscoRipper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java index dca72913..d0a36cdc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java @@ -116,7 +116,7 @@ public class VscoRipper extends AbstractHTMLRipper { } private String getUserName() { - Pattern p = Pattern.compile("^https?://vsco.co/([a-zA-Z0-9-]+)/gallery(/)?"); + Pattern p = Pattern.compile("^https?://vsco.co/([a-zA-Z0-9-]+)(/gallery)?(/)?"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { @@ -200,7 +200,7 @@ public class VscoRipper extends AbstractHTMLRipper { } //Member profile (Usernames should all be different, so this should work. - p = Pattern.compile("^https?://vsco.co/([a-zA-Z0-9-]+)/gallery(/)?"); + p = Pattern.compile("^https?://vsco.co/([a-zA-Z0-9-]+)(/gallery)?(/)?"); m = p.matcher(url.toExternalForm()); if (m.matches()){ From 57aa76bf4db989d041db40fdaa6f9a356f55398b Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 28 Mar 2020 14:14:52 -0400 Subject: [PATCH 018/512] 1.7.92: Added read-comic.com ripper; Fix Pawoo ripper; Add ChineseSimplified language file; Fixed artstation ripper --- pom.xml | 2 +- ripme.json | 5 +++-- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 408d0fd4..4a876b98 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.rarchives.ripme ripme jar - 1.7.91 + 1.7.92 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 6e0d56a8..e80f6b55 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "currentHash": "fb1e97064f255315da31b794897a9aacf2f958aad89202677deb6cdcf565ec77", + "currentHash": "e0d946933c0a89da03e9a5a00420a87dd8c2fc23eae01e1a2fb8390eceb9b868", "changeList": [ + "1.7.92: Added read-comic.com ripper; Fix Pawoo ripper; Add ChineseSimplified language file; Fixed artstation ripper", "1.7.91: Fixed luscious ripper. Fixed VK ripper; Added Kingcomix ripper", "1.7.90: Added FitnakedgirlsRipper; Fixed VK Album Ripper; Fixed Myreadingmanga Ripper; Fixed windows max file name; Fixed Pornhub Video Ripper; Fixed Motherless Ripper; Fixed Instagram Ripper", "1.7.89: Improved twitter ripper; Fixed xhamster image ripper; Fixed allporncomic ripper; Added Ripper for folio.ink", @@ -263,5 +264,5 @@ "1.0.3: Added VK.com ripper", "1.0.1: Added auto-update functionality" ], - "latestVersion": "1.7.91" + "latestVersion": "1.7.92" } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 86760f1e..a09730ce 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -23,7 +23,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.91"; + private static final String DEFAULT_VERSION = "1.7.92"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static String mainFileName; From d6ffcdfbe7def0f2b637b6600a3031a929cb421a Mon Sep 17 00:00:00 2001 From: Felix Friebe Date: Sun, 5 Apr 2020 02:21:55 +0200 Subject: [PATCH 019/512] Fix MotherlessRipper: change CSS selector --- .../com/rarchives/ripme/ripper/rippers/MotherlessRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java index 7bb8451a..598cf5d4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java @@ -81,7 +81,7 @@ public class MotherlessRipper extends AbstractHTMLRipper { protected List getURLsFromPage(Document page) { List pageURLs = new ArrayList<>(); - for (Element thumb : page.select("div.thumb a.img-container")) { + for (Element thumb : page.select("div.thumb-container a.img-container")) { if (isStopped()) { break; } From e814e0106215257d9dea541a384e5db5340536d7 Mon Sep 17 00:00:00 2001 From: BlackBirdd Date: Thu, 26 Mar 2020 20:38:41 +0100 Subject: [PATCH 020/512] Fixed e621 ripper --- .../ripme/ripper/rippers/E621Ripper.java | 49 +++++++++++++++---- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java index 534a1d0d..dfb0468d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java @@ -16,6 +16,11 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +// updated by BlackBirdd (https://github.com/blackbirddx) after e621 update in March 2020 +// old url style => new url style: +// /post/index/1/ => /posts?tags= +// /pool/show/ => /pools/id + public class E621Ripper extends AbstractHTMLRipper { private static final Logger logger = Logger.getLogger(E621Ripper.class); @@ -23,6 +28,9 @@ public class E621Ripper extends AbstractHTMLRipper { private static Pattern gidPattern2 = null; private static Pattern gidPatternPool = null; + private static Pattern gidPatternNew = null; + private static Pattern gidPatternPoolNew = null; + private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621"); public E621Ripper(URL url) throws IOException { @@ -46,15 +54,15 @@ public class E621Ripper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - if (url.getPath().startsWith("/pool/show/")) - return Http.url("https://e621.net/pool/show/" + getTerm(url)).get(); + if (url.getPath().startsWith("/pool")) + return Http.url("https://e621.net/pools/" + getTerm(url)).get(); else - return Http.url("https://e621.net/post/index/1/" + getTerm(url)).get(); + return Http.url("https://e621.net/posts?tags=" + getTerm(url)).get(); } @Override public List getURLsFromPage(Document page) { - Elements elements = page.select("div > span.thumb > a"); + Elements elements = page.select("article > a"); List res = new ArrayList<>(); for (Element e : elements) { @@ -68,8 +76,8 @@ public class E621Ripper extends AbstractHTMLRipper { @Override public Document getNextPage(Document page) throws IOException { - if (!page.select("a.next_page").isEmpty()) { - return Http.url(page.select("a.next_page").attr("abs:href")).get(); + if (!page.select("a#paginator-next").isEmpty()) { + return Http.url(page.select("a#paginator-next").attr("abs:href")).get(); } else { throw new IOException("No more pages."); } @@ -88,6 +96,10 @@ public class E621Ripper extends AbstractHTMLRipper { if (gidPatternPool == null) gidPatternPool = Pattern.compile( "^https?://(www\\.)?e621\\.net/pool/show/([a-zA-Z0-9$_.+!*'(),%:\\-]+)(\\?.*)?(/.*)?(#.*)?$"); + if (gidPatternNew == null) + gidPatternNew = Pattern.compile("^https?://(www\\.)?e621\\.net/posts\\?tags=([\\S]+)"); + if (gidPatternPoolNew == null) + gidPatternPoolNew = Pattern.compile("^https?://(www\\.)?e621\\.net/pools/([\\d]+)"); Matcher m = gidPattern.matcher(url.toExternalForm()); if (m.matches()) { @@ -100,14 +112,26 @@ public class E621Ripper extends AbstractHTMLRipper { return m.group(2); } + m = gidPatternNew.matcher(url.toExternalForm()); + if (m.matches()) { + LOGGER.info(m.group(2)); + return m.group(2); + } + + m = gidPatternPoolNew.matcher(url.toExternalForm()); + if (m.matches()) { + LOGGER.info(m.group(2)); + return m.group(2); + } + throw new MalformedURLException( - "Expected e621.net URL format: e621.net/post/index/1/searchterm - got " + url + " instead"); + "Expected e621.net URL format: e621.net/posts?tags=searchterm - got " + url + " instead"); } @Override public String getGID(URL url) throws MalformedURLException { String prefix = ""; - if (url.getPath().startsWith("/pool/show/")) { + if (url.getPath().startsWith("/pool")) { prefix = "pool_"; } return Utils.filesystemSafe(prefix + getTerm(url)); @@ -150,7 +174,7 @@ public class E621Ripper extends AbstractHTMLRipper { private String getFullSizedImage(URL imageURL) throws IOException { Document page = Http.url(imageURL).retries(3).get(); - Elements video = page.select("video > source"); + /*Elements video = page.select("video > source"); Elements flash = page.select("embed"); Elements image = page.select("a#highres"); if (video.size() > 0) { @@ -161,8 +185,13 @@ public class E621Ripper extends AbstractHTMLRipper { return image.attr("href"); } else { throw new IOException(); - } + }*/ + if (!page.select("div#image-download-link > a").isEmpty()) { + return page.select("div#image-download-link > a").attr("abs:href"); + } else { + throw new IOException(); + } } } From 03abd937f3bfefce44ac85fd95b4d630f36d3164 Mon Sep 17 00:00:00 2001 From: BlackBirdd Date: Fri, 27 Mar 2020 09:40:49 +0100 Subject: [PATCH 021/512] fix e621 ripper regex for urls with more arguments --- .../java/com/rarchives/ripme/ripper/rippers/E621Ripper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java index dfb0468d..93354e5f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java @@ -97,9 +97,9 @@ public class E621Ripper extends AbstractHTMLRipper { gidPatternPool = Pattern.compile( "^https?://(www\\.)?e621\\.net/pool/show/([a-zA-Z0-9$_.+!*'(),%:\\-]+)(\\?.*)?(/.*)?(#.*)?$"); if (gidPatternNew == null) - gidPatternNew = Pattern.compile("^https?://(www\\.)?e621\\.net/posts\\?tags=([\\S]+)"); + gidPatternNew = Pattern.compile("^https?://(www\\.)?e621\\.net/posts\\?tags=([a-zA-Z0-9$_.+!*'(),%:\\-]+)(\\&[\\S]+)?"); if (gidPatternPoolNew == null) - gidPatternPoolNew = Pattern.compile("^https?://(www\\.)?e621\\.net/pools/([\\d]+)"); + gidPatternPoolNew = Pattern.compile("^https?://(www\\.)?e621\\.net/pools/([\\d]+)(\\?[\\S]*)?"); Matcher m = gidPattern.matcher(url.toExternalForm()); if (m.matches()) { From fec50e465e169461e5afd891750aef4eef3c0f17 Mon Sep 17 00:00:00 2001 From: BlackBirdd Date: Wed, 8 Apr 2020 15:06:39 +0200 Subject: [PATCH 022/512] add e621 cloudflare captcha cookies --- .../ripme/ripper/rippers/E621Ripper.java | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java index 93354e5f..1b4e2bb8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java @@ -3,12 +3,16 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; +import com.rarchives.ripme.ui.RipStatusMessage; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.log4j.Logger; @@ -16,10 +20,7 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; -// updated by BlackBirdd (https://github.com/blackbirddx) after e621 update in March 2020 -// old url style => new url style: -// /post/index/1/ => /posts?tags= -// /pool/show/ => /pools/id + public class E621Ripper extends AbstractHTMLRipper { private static final Logger logger = Logger.getLogger(E621Ripper.class); @@ -33,8 +34,21 @@ public class E621Ripper extends AbstractHTMLRipper { private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621"); + private Map cookies = new HashMap(); + private boolean useAPIRipper = false; + public E621Ripper(URL url) throws IOException { super(url); + loadConfig(); + } + + private void loadConfig() { + String cookiesString = Utils.getConfigString("e621.captcha_cookies", ""); + if(!cookiesString.equals(" ")) { + cookies = RipUtils.getCookiesFromString(cookiesString); + sendUpdate(RipStatusMessage.STATUS.RIP_ERRORED, "Using CloudFlare captcha cookies, make sure to update them!"); + } + useAPIRipper = Utils.getConfigBoolean("e621.use_API_ripper", false); } @Override @@ -55,9 +69,9 @@ public class E621Ripper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { if (url.getPath().startsWith("/pool")) - return Http.url("https://e621.net/pools/" + getTerm(url)).get(); + return Http.url("https://e621.net/pools/" + getTerm(url)).cookies(cookies).get(); else - return Http.url("https://e621.net/posts?tags=" + getTerm(url)).get(); + return Http.url("https://e621.net/posts?tags=" + getTerm(url)).cookies(cookies).get(); } @Override @@ -77,7 +91,7 @@ public class E621Ripper extends AbstractHTMLRipper { @Override public Document getNextPage(Document page) throws IOException { if (!page.select("a#paginator-next").isEmpty()) { - return Http.url(page.select("a#paginator-next").attr("abs:href")).get(); + return Http.url(page.select("a#paginator-next").attr("abs:href")).cookies(cookies).get(); } else { throw new IOException("No more pages."); } @@ -90,6 +104,9 @@ public class E621Ripper extends AbstractHTMLRipper { } private String getTerm(URL url) throws MalformedURLException { + // old url style => new url style: + // /post/index/1/ => /posts?tags= + // /pool/show/ => /pools/id if (gidPattern == null) gidPattern = Pattern.compile( "^https?://(www\\.)?e621\\.net/post/index/[^/]+/([a-zA-Z0-9$_.+!*'():,%\\-]+)(/.*)?(#.*)?$"); @@ -173,7 +190,7 @@ public class E621Ripper extends AbstractHTMLRipper { } private String getFullSizedImage(URL imageURL) throws IOException { - Document page = Http.url(imageURL).retries(3).get(); + Document page = Http.url(imageURL).cookies(cookies).retries(3).get(); /*Elements video = page.select("video > source"); Elements flash = page.select("embed"); Elements image = page.select("a#highres"); From 56f0aa3da30ee59e33276cae9a26fbb066f2ebf8 Mon Sep 17 00:00:00 2001 From: BlackBirdd Date: Wed, 8 Apr 2020 19:58:33 +0200 Subject: [PATCH 023/512] update e621 cookies --- .../ripme/ripper/rippers/E621Ripper.java | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java index 1b4e2bb8..e3013bd7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java @@ -35,20 +35,23 @@ public class E621Ripper extends AbstractHTMLRipper { private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621"); private Map cookies = new HashMap(); - private boolean useAPIRipper = false; public E621Ripper(URL url) throws IOException { super(url); - loadConfig(); } - private void loadConfig() { - String cookiesString = Utils.getConfigString("e621.captcha_cookies", ""); - if(!cookiesString.equals(" ")) { + private void loadCookies() { + String cookiesString = Utils.getConfigString("e621.cookies", ""); + if(!cookiesString.equals("")) { cookies = RipUtils.getCookiesFromString(cookiesString); - sendUpdate(RipStatusMessage.STATUS.RIP_ERRORED, "Using CloudFlare captcha cookies, make sure to update them!"); + if(cookies.containsKey("cf_clearance")) + sendUpdate(RipStatusMessage.STATUS.RIP_ERRORED, "Using CloudFlare captcha cookies, make sure to update them in config!"); } - useAPIRipper = Utils.getConfigBoolean("e621.use_API_ripper", false); + } + + private void warnAboutBlacklist(Document page) { + if(!page.select("div.hidden-posts-notice").isEmpty()) + sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_WARN, "Some posts are blacklisted. Consider logging in. Search for \"e621\" in this wiki page: https://github.com/RipMeApp/ripme/wiki/Config-options"); } @Override @@ -68,10 +71,15 @@ public class E621Ripper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { + loadCookies(); + Document page; if (url.getPath().startsWith("/pool")) - return Http.url("https://e621.net/pools/" + getTerm(url)).cookies(cookies).get(); + page = Http.url("https://e621.net/pools/" + getTerm(url)).cookies(cookies).get(); else - return Http.url("https://e621.net/posts?tags=" + getTerm(url)).cookies(cookies).get(); + page = Http.url("https://e621.net/posts?tags=" + getTerm(url)).cookies(cookies).get(); + + warnAboutBlacklist(page); + return page; } @Override @@ -90,6 +98,7 @@ public class E621Ripper extends AbstractHTMLRipper { @Override public Document getNextPage(Document page) throws IOException { + warnAboutBlacklist(page); if (!page.select("a#paginator-next").isEmpty()) { return Http.url(page.select("a#paginator-next").attr("abs:href")).cookies(cookies).get(); } else { @@ -207,6 +216,8 @@ public class E621Ripper extends AbstractHTMLRipper { if (!page.select("div#image-download-link > a").isEmpty()) { return page.select("div#image-download-link > a").attr("abs:href"); } else { + if(!page.select("#blacklist-box").isEmpty()) + sendUpdate(RipStatusMessage.STATUS.RIP_ERRORED, "Cannot download image - blocked by blacklist. Consider logging in. Search for \"e621\" in this wiki page: https://github.com/RipMeApp/ripme/wiki/Config-options"); throw new IOException(); } } From 52b7e9443d91c656e1b8cbf2b71a85b201410b81 Mon Sep 17 00:00:00 2001 From: BlackBirdd Date: Thu, 9 Apr 2020 14:04:12 +0200 Subject: [PATCH 024/512] e621 fix regex --- .../ripme/ripper/rippers/E621Ripper.java | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java index e3013bd7..bac6b51f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java @@ -6,6 +6,8 @@ import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; import com.rarchives.ripme.ui.RipStatusMessage; +import com.rarchives.ripme.ui.RipStatusMessage.STATUS; + import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -35,23 +37,36 @@ public class E621Ripper extends AbstractHTMLRipper { private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621"); private Map cookies = new HashMap(); + private String userAgent = USER_AGENT; public E621Ripper(URL url) throws IOException { super(url); } - private void loadCookies() { + private void loadConfig() { String cookiesString = Utils.getConfigString("e621.cookies", ""); if(!cookiesString.equals("")) { cookies = RipUtils.getCookiesFromString(cookiesString); if(cookies.containsKey("cf_clearance")) - sendUpdate(RipStatusMessage.STATUS.RIP_ERRORED, "Using CloudFlare captcha cookies, make sure to update them in config!"); + sendUpdate(STATUS.DOWNLOAD_WARN, "Using CloudFlare captcha cookies, make sure to update them and set your browser's useragent in config!"); + if(cookies.containsKey("remember")) + sendUpdate(STATUS.DOWNLOAD_WARN, "Logging in using auth cookie."); } + userAgent = Utils.getConfigString("e621.useragent", USER_AGENT); + } private void warnAboutBlacklist(Document page) { if(!page.select("div.hidden-posts-notice").isEmpty()) - sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_WARN, "Some posts are blacklisted. Consider logging in. Search for \"e621\" in this wiki page: https://github.com/RipMeApp/ripme/wiki/Config-options"); + sendUpdate(STATUS.DOWNLOAD_WARN, "Some posts are blacklisted. Consider logging in. Search for \"e621\" in this wiki page: https://github.com/RipMeApp/ripme/wiki/Config-options"); + } + + private Document getDocument(String url, int retries) throws IOException { + return Http.url(url).userAgent(userAgent).retries(retries).cookies(cookies).get(); + } + + private Document getDocument(String url) throws IOException { + return getDocument(url, 1); } @Override @@ -71,12 +86,12 @@ public class E621Ripper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - loadCookies(); + loadConfig(); Document page; if (url.getPath().startsWith("/pool")) - page = Http.url("https://e621.net/pools/" + getTerm(url)).cookies(cookies).get(); + page = getDocument("https://e621.net/pools/" + getTerm(url)); else - page = Http.url("https://e621.net/posts?tags=" + getTerm(url)).cookies(cookies).get(); + page = getDocument("https://e621.net/posts?tags=" + getTerm(url)); warnAboutBlacklist(page); return page; @@ -100,7 +115,7 @@ public class E621Ripper extends AbstractHTMLRipper { public Document getNextPage(Document page) throws IOException { warnAboutBlacklist(page); if (!page.select("a#paginator-next").isEmpty()) { - return Http.url(page.select("a#paginator-next").attr("abs:href")).cookies(cookies).get(); + return getDocument(page.select("a#paginator-next").attr("abs:href")); } else { throw new IOException("No more pages."); } @@ -123,7 +138,7 @@ public class E621Ripper extends AbstractHTMLRipper { gidPatternPool = Pattern.compile( "^https?://(www\\.)?e621\\.net/pool/show/([a-zA-Z0-9$_.+!*'(),%:\\-]+)(\\?.*)?(/.*)?(#.*)?$"); if (gidPatternNew == null) - gidPatternNew = Pattern.compile("^https?://(www\\.)?e621\\.net/posts\\?tags=([a-zA-Z0-9$_.+!*'(),%:\\-]+)(\\&[\\S]+)?"); + gidPatternNew = Pattern.compile("^https?://(www\\.)?e621\\.net/posts\\?([\\S]*?)tags=([a-zA-Z0-9$_.+!*'(),%:\\-]+)(\\&[\\S]+)?"); if (gidPatternPoolNew == null) gidPatternPoolNew = Pattern.compile("^https?://(www\\.)?e621\\.net/pools/([\\d]+)(\\?[\\S]*)?"); @@ -140,8 +155,8 @@ public class E621Ripper extends AbstractHTMLRipper { m = gidPatternNew.matcher(url.toExternalForm()); if (m.matches()) { - LOGGER.info(m.group(2)); - return m.group(2); + LOGGER.info(m.group(3)); + return m.group(3); } m = gidPatternPoolNew.matcher(url.toExternalForm()); @@ -199,7 +214,7 @@ public class E621Ripper extends AbstractHTMLRipper { } private String getFullSizedImage(URL imageURL) throws IOException { - Document page = Http.url(imageURL).cookies(cookies).retries(3).get(); + Document page = getDocument(imageURL.toExternalForm(), 3); /*Elements video = page.select("video > source"); Elements flash = page.select("embed"); Elements image = page.select("a#highres"); From 80cf9db90ba28e0e27ef1e080e5044bea69e7003 Mon Sep 17 00:00:00 2001 From: LameLemon Date: Sat, 25 Apr 2020 16:04:39 +0100 Subject: [PATCH 025/512] Updated pt_PT translation --- .../resources/LabelsBundle_pt_PT.properties | 62 ++++++++++++------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/src/main/resources/LabelsBundle_pt_PT.properties b/src/main/resources/LabelsBundle_pt_PT.properties index 3dcbafe1..a0058524 100644 --- a/src/main/resources/LabelsBundle_pt_PT.properties +++ b/src/main/resources/LabelsBundle_pt_PT.properties @@ -4,9 +4,9 @@ created = criado modified = modificado queue = Fila Configuration = Configuração +open = Abrir # Keys for the Configuration menu - current.version = Versão atual check.for.updates = Verificar atualizações auto.update = Atualização automática? @@ -27,31 +27,49 @@ restore.window.position = Restaurar posição da janela remember.url.history = Lembrar histórico de URL loading.history.from = Carregar histórico de -# Misc UI keys +# Queue keys +queue.remove.all = Remover todos +queue.validation = Tem a certeza de que quer remover todos os elementos da fila? +queue.remove.selected = Remover seleccionados +# History +re-rip.checked = Re-rip seleccionados +remove = Remover +clear = Limpar +history.check.all = Seleccionar todos +history.check.none = Não seleccionar nenhuma +history.check.selected = Marca seleccionada +history.uncheck.selected = Desmaracar seleccionada +history.load.failed.warning = RipMe não carregou o ficheiro de histórico no historyFile.getAbsolutePath() \n\nErro: %s\n\nFechar o RipMe sobregravará automaticamente o conteúdo deste ficheiro.\nPode querer fazer o backup do ficheiro antes de fechar o RipMe! +history.load.none = Não há entradas no histórico para re-rip. Rip alguns álbuns primeiro. +history.load.none.checked = Não foram seleccionadas entradas no histórico. Selecione uma entrada clicando na caixa à direita do URL ou clique com o botão direito em um URL para selecione/desselecione todos os itens. + +# TrayIcon +tray.show = Mostrar +tray.hide = Esconder +tray.autorip = AutoRip clipboard +tray.exit = Saída + +# Misc UI keys loading.history.from.configuration = A carregar o histórico da configuração interrupted.while.waiting.to.rip.next.album = Interrompido durante a espera do rip do próximo álbum inactive = Inativo -re-rip.checked = Re-rip verificado -remove = Remover -clear = Limpar -download.url.list = Download url list -select.save.dir = Select Save Directory +download.url.list = A lista de URLs para download +select.save.dir = Selecione salvar diretório # Keys for the logs generated by DownloadFileThread - -nonretriable.status.code = Non-retriable status code -retriable.status.code = Retriable status code -server.doesnt.support.resuming.downloads = Server doesn't support resuming downloads +nonretriable.status.code = Código de estado não recuperável +retriable.status.code = Código de estado recuperável +server.doesnt.support.resuming.downloads = Servidor não suporta o reinício de downloads # A "magic number" can also be called a file signature -was.unable.to.get.content.type.using.magic.number = Was unable to get content type using magic number -magic.number.was = Magic number was -deleting.existing.file = Deleting existing file -request.properties = Request properties -download.interrupted = Download interrupted -exceeded.maximum.retries = Exceeded maximum retries -http.status.exception = HTTP status exception -exception.while.downloading.file = Exception while downloading file -failed.to.download = Failed to download -skipping = Skipping -file.already.exists = file already exists +was.unable.to.get.content.type.using.magic.number = Impossibilidade de obter o tipo de conteúdo utilizando o número mágico +magic.number.was = Número mágico foi +deleting.existing.file = Eliminação do ficheiro existente +request.properties = Propriedades do pedido +download.interrupted = Download interrompido +exceeded.maximum.retries = Excedeu tentativas máximas +http.status.exception = Exceção de status HTTP +exception.while.downloading.file = Exceção enquanto o ficheiro era baixado +failed.to.download = Falha no download +skipping = Pulando +file.already.exists = Ficheiro já existe \ No newline at end of file From 7d466a0516a0769469071e3828a35d439b741cf3 Mon Sep 17 00:00:00 2001 From: BlackBirdd Date: Sun, 26 Apr 2020 21:31:27 +0200 Subject: [PATCH 026/512] add e621 test --- .../tst/ripper/rippers/E621RipperTest.java | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java index 5f7165b2..41a3e33e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java @@ -8,16 +8,43 @@ import org.junit.jupiter.api.Test; public class E621RipperTest extends RippersTest { public void testRip() throws IOException { - E621Ripper ripper = new E621Ripper(new URL("https://e621.net/post/index/1/beach")); + E621Ripper ripper = new E621Ripper(new URL("https://e621.net/posts?tags=beach")); testRipper(ripper); } @Test public void testFlashOrWebm() throws IOException { - E621Ripper ripper = new E621Ripper(new URL("https://e621.net/post/index/1/gif")); + E621Ripper ripper = new E621Ripper(new URL("https://e621.net/posts?page=4&tags=gif+rating%3As+3d")); testRipper(ripper); } @Test public void testGetNextPage() throws IOException { + E621Ripper nextPageRipper = new E621Ripper(new URL("https://e621.net/posts?tags=cosmicminerals")); + try { + nextPageRipper.getNextPage(nextPageRipper.getFirstPage()); + assert (true); + } catch (IOException e) { + throw e; + } + + E621Ripper noNextPageRipper = new E621Ripper(new URL("https://e621.net/post/index/1/cosmicminerals")); + try { + noNextPageRipper.getNextPage(noNextPageRipper.getFirstPage()); + } catch (IOException e) { + assertEquals(e.getMessage(), "No more pages."); + } + } + @Test + public void testOldRip() throws IOException { + E621Ripper ripper = new E621Ripper(new URL("https://e621.net/post/index/1/beach")); + testRipper(ripper); + } + @Test + public void testOldFlashOrWebm() throws IOException { + E621Ripper ripper = new E621Ripper(new URL("https://e621.net/post/index/1/gif")); + testRipper(ripper); + } + @Test + public void testOldGetNextPage() throws IOException { E621Ripper nextPageRipper = new E621Ripper(new URL("https://e621.net/post/index/1/cosmicminerals")); try { nextPageRipper.getNextPage(nextPageRipper.getFirstPage()); From 7fe7f7a284e571c3fb82b62f248b59effe77cd85 Mon Sep 17 00:00:00 2001 From: darshan-poudel Date: Fri, 1 May 2020 11:08:49 -0500 Subject: [PATCH 027/512] #38 comment-589679044: Added a ripper for freecomiconline.me supports chapter download and continues to next chapter until caught up --- .../ripper/rippers/FreeComicOnlineRipper.java | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/FreeComicOnlineRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FreeComicOnlineRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FreeComicOnlineRipper.java new file mode 100644 index 00000000..c640e6ec --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FreeComicOnlineRipper.java @@ -0,0 +1,89 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class FreeComicOnlineRipper extends AbstractHTMLRipper { + + public FreeComicOnlineRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "freecomiconline"; + } + + @Override + public String getDomain() { + return "freecomiconline.me"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https://freecomiconline.me/comic/([a-zA-Z0-9_\\-]+)/([a-zA-Z0-9_\\-]+)/?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1) + "_" + m.group(2); + } + p = Pattern.compile("^https://freecomiconline.me/comic/([a-zA-Z0-9_\\-]+)/?$"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected freecomiconline URL format: " + + "freecomiconline.me/TITLE/CHAPTER - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public Document getNextPage(Document doc) throws IOException { + String nextPage = doc.select("div.select-pagination a").get(1).attr("href"); + String nextUrl = ""; + // "https://freecomiconline.me/comic/([a-zA-Z0-9_\\-]+)/([a-zA-Z0-9_\\-]+)/?$" + System.out.println("\n\nPagination.(0).href: "+ nextPage); + + Pattern p = Pattern.compile("https://freecomiconline.me/comic/([a-zA-Z0-9_\\-]+)/([a-zA-Z0-9_\\-]+)/?$"); + Matcher m = p.matcher(nextPage); + + if(m.matches()){ + nextUrl = m.group(0); + //System.out.println("\n\nMatched and recreatedUrl: "+ nextUrl+"\n\n"); + + } + if(nextUrl.equals("")) throw new IOException("No more pages"); + + sleep(500); + return Http.url(nextUrl).get(); + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + for (Element el : doc.select(".wp-manga-chapter-img")) { + result.add(el.attr("src")); + } + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} From 1ea679575c82323e3f014453f45a603cfeaf1e45 Mon Sep 17 00:00:00 2001 From: darshan-poudel Date: Fri, 1 May 2020 11:15:42 -0500 Subject: [PATCH 028/512] cleaned up comments --- .../ripme/ripper/rippers/FreeComicOnlineRipper.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FreeComicOnlineRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FreeComicOnlineRipper.java index c640e6ec..daba6a41 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FreeComicOnlineRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FreeComicOnlineRipper.java @@ -7,10 +7,8 @@ import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; - import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; - import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; @@ -56,19 +54,12 @@ public class FreeComicOnlineRipper extends AbstractHTMLRipper { public Document getNextPage(Document doc) throws IOException { String nextPage = doc.select("div.select-pagination a").get(1).attr("href"); String nextUrl = ""; - // "https://freecomiconline.me/comic/([a-zA-Z0-9_\\-]+)/([a-zA-Z0-9_\\-]+)/?$" - System.out.println("\n\nPagination.(0).href: "+ nextPage); - Pattern p = Pattern.compile("https://freecomiconline.me/comic/([a-zA-Z0-9_\\-]+)/([a-zA-Z0-9_\\-]+)/?$"); Matcher m = p.matcher(nextPage); - if(m.matches()){ nextUrl = m.group(0); - //System.out.println("\n\nMatched and recreatedUrl: "+ nextUrl+"\n\n"); - } if(nextUrl.equals("")) throw new IOException("No more pages"); - sleep(500); return Http.url(nextUrl).get(); } From 606f3665ef7bcdb7eeda8b9064164d0dc21a3e28 Mon Sep 17 00:00:00 2001 From: darshan-poudel Date: Fri, 1 May 2020 12:32:55 -0500 Subject: [PATCH 029/512] Fixing indent --- .../rarchives/ripme/ripper/rippers/FreeComicOnlineRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FreeComicOnlineRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FreeComicOnlineRipper.java index daba6a41..e08d77fd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FreeComicOnlineRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FreeComicOnlineRipper.java @@ -57,7 +57,7 @@ public class FreeComicOnlineRipper extends AbstractHTMLRipper { Pattern p = Pattern.compile("https://freecomiconline.me/comic/([a-zA-Z0-9_\\-]+)/([a-zA-Z0-9_\\-]+)/?$"); Matcher m = p.matcher(nextPage); if(m.matches()){ - nextUrl = m.group(0); + nextUrl = m.group(0); } if(nextUrl.equals("")) throw new IOException("No more pages"); sleep(500); From bf8f3866313468618cec9c7019e43ad3a993e670 Mon Sep 17 00:00:00 2001 From: Version Kinda Date: Sat, 2 May 2020 09:10:19 +0530 Subject: [PATCH 030/512] changed imagefap links from http to https #1629 #1387 #1074 #1065 --- .../com/rarchives/ripme/ripper/rippers/ImagefapRipper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java index f097e667..b1a27b47 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java @@ -38,7 +38,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { @Override public URL sanitizeURL(URL url) throws MalformedURLException { String gid = getGID(url); - String newURL = "http://www.imagefap.com/gallery.php?"; + String newURL = "https://www.imagefap.com/gallery.php?"; if (isNewAlbumType) { newURL += "p"; } @@ -107,7 +107,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { String nextURL = null; for (Element a : doc.select("a.link3")) { if (a.text().contains("next")) { - nextURL = "http://imagefap.com/gallery.php" + a.attr("href"); + nextURL = "https://imagefap.com/gallery.php" + a.attr("href"); break; } } From 1d600e5a31d0c14d31850aab6fe860e562f18a87 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 20 May 2020 23:55:38 -0400 Subject: [PATCH 031/512] Now sends a referrer when fetching first page --- .../com/rarchives/ripme/ripper/rippers/GfycatRipper.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java index 16205115..37b2d5ae 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java @@ -25,6 +25,7 @@ public class GfycatRipper extends AbstractHTMLRipper { String username = ""; String cursor = ""; String count = "30"; + String REFERRER = "www.reddit.com"; @@ -64,10 +65,10 @@ public class GfycatRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { if (!isProfile()) { - return Http.url(url).get(); + return Http.url(url).referrer(REFERRER).get(); } else { username = getGID(url); - return Http.url(new URL("https://api.gfycat.com/v1/users/" + username + "/gfycats")).ignoreContentType().get(); + return Http.url(new URL("https://api.gfycat.com/v1/users/" + username + "/gfycats")).referrer((REFERRER)).ignoreContentType().get(); } } From b5ab57fed3641f536aa92fe182fa6357205dcdbc Mon Sep 17 00:00:00 2001 From: Rezha Julio Date: Sat, 23 May 2020 15:20:46 +0700 Subject: [PATCH 032/512] Fix malformed log --- .../java/com/rarchives/ripme/ripper/DownloadFileThread.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 3613273e..ba1104eb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -84,7 +84,7 @@ class DownloadFileThread extends Thread { logger.info("[!] " + Utils.getLocalizedString("deleting.existing.file") + prettySaveAs); saveAs.delete(); } else { - logger.info("[!] " + Utils.getLocalizedString("skipping") + url + " -- " + logger.info("[!] " + Utils.getLocalizedString("skipping") + " " + url + " -- " + Utils.getLocalizedString("file.already.exists") + ": " + prettySaveAs); observer.downloadExists(url, saveAs); return; From 9a013c74ee066e6121df6d3c7098502e9e1ed382 Mon Sep 17 00:00:00 2001 From: borderline232 Date: Sat, 23 May 2020 14:37:10 -0400 Subject: [PATCH 033/512] RedgifsRipper: Implemented redgifs Ripper - Using similar functionality found in the gfycat ripper the same was done for the redgifs ripper - A user's profile can be ripped, by retrieving all gfycats from their user api call - The categories and search ripper is also implemented but uses and incrementing start and count in the api url paramters as the cursor requires cookies (I believe) to work and this one just increments until there are no gfycats and returns error (there are no more gfycats) - Differentiated between getting profile and search urls using regex matching --- .../ripme/ripper/rippers/RedgifsRipper.java | 186 ++++++++++++++++++ .../tst/ripper/rippers/RedgifsRipperTest.java | 48 +++++ 2 files changed, 234 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java new file mode 100644 index 00000000..9d6152b9 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java @@ -0,0 +1,186 @@ +package com.rarchives.ripme.ripper.rippers; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; +import org.json.JSONArray; +import org.json.JSONObject; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class RedgifsRipper extends AbstractHTMLRipper { + + private static final String HOST = "redgifs.com"; + String username = ""; + String cursor = ""; + String count = "100"; + + String searchText = ""; + int searchCount = 150; + int searchStart = 0; + + public RedgifsRipper(URL url) throws IOException { + super(new URL(url.toExternalForm().replace("thumbs.", ""))); + } + + @Override + public String getDomain() { return "redgifs.com"; } + + @Override + public String getHost() { + return "redgifs"; + } + + @Override + public boolean canRip(URL url) { + return url.getHost().endsWith(HOST); + } + + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + String sUrl = url.toExternalForm(); + sUrl = sUrl.replace("/gifs/detail", ""); + sUrl = sUrl.replace("/amp", ""); + return new URL(sUrl); + } + + public Matcher isProfile() { + Pattern p = Pattern.compile("^https?://[wm.]*redgifs\\.com/users/([a-zA-Z0-9_-]+).*$"); + return p.matcher(url.toExternalForm()); + } + + public Matcher isSearch() { + Pattern p = Pattern.compile("^https?://[wm.]*redgifs\\.com/gifs/browse/([a-zA-Z0-9_-]+).*$"); + return p.matcher(url.toExternalForm()); + } + + public Matcher isSingleton() { + Pattern p = Pattern.compile("^https?://[wm.]*redgifs\\.com/watch/([a-zA-Z0-9_-]+).*$"); + return p.matcher(url.toExternalForm()); + } + + @Override + public Document getFirstPage() throws IOException { + if (!isProfile().matches() && !isSearch().matches()) { + return Http.url(url).get(); + } else if (isSearch().matches()) { + searchText = getGID(url).replace("-", " "); + return Http.url( + new URL("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart*searchCount)).ignoreContentType().get(); + } else { + username = getGID(url); + return Http.url(new URL("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count)) + .ignoreContentType().get(); + } + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + + @Override + public String getGID(URL url) throws MalformedURLException { + + Matcher m = isProfile(); + if (m.matches()) { + return m.group(1); + } + m = isSearch(); + if (m.matches()) { + return m.group(1); + } + m = isSingleton(); + if (m.matches()) { + return m.group(1).split("-")[0]; + } + throw new MalformedURLException( + "Expected redgifs.com format: " + + "redgifs.com/id or " + + "thumbs.redgifs.com/id.gif" + + " Got: " + url); + } + + private String stripHTMLTags(String t) { + t = t.replaceAll("\n" + + " \n" + + " ", ""); + t = t.replaceAll("\n" + + "", ""); + t = t.replaceAll("\n", ""); + t = t.replaceAll("=\"\"", ""); + return t; + } + + @Override + public Document getNextPage(Document doc) throws IOException { + if (!isProfile().matches()) { + return Http.url( + new URL("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + + "&count=" + searchCount + "&start=" + searchCount*++searchStart)) + .ignoreContentType().get(); + } else { + if (cursor.equals("")) { + throw new IOException("No more pages"); + } else { + return Http.url(new URL("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor)).ignoreContentType().get(); + } + } + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + if (isProfile().matches() || isSearch().matches()) { + JSONObject page = new JSONObject(stripHTMLTags(doc.html())); + JSONArray content = page.getJSONArray("gfycats"); + for (int i = 0; i < content.length(); i++) { + result.add(content.getJSONObject(i).getString("mp4Url")); + } + cursor = page.getString("cursor"); + } else { + Elements videos = doc.select("script"); + for (Element el : videos) { + String json = el.html(); + if (json.startsWith("{")) { + JSONObject page = new JSONObject(json); + result.add(page.getJSONObject("video").getString("contentUrl")); + } + } + } + return result; + } + + /** + * Helper method for retrieving video URLs. + * @param url URL to gfycat page + * @return URL to video + * @throws IOException + */ + public static String getVideoURL(URL url) throws IOException { + LOGGER.info("Retrieving " + url.toExternalForm()); + + //Sanitize the URL first + url = new URL(url.toExternalForm().replace("/gifs/detail", "")); + + Document doc = Http.url(url).get(); + Elements videos = doc.select("script"); + for (Element el : videos) { + String json = el.html(); + if (json.startsWith("{")) { + JSONObject page = new JSONObject(json); + return page.getJSONObject("video").getString("contentUrl"); + } + } + throw new IOException(); + } + +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java new file mode 100644 index 00000000..46624303 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java @@ -0,0 +1,48 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.GfycatRipper; +import com.rarchives.ripme.ripper.rippers.RedgifsRipper; +import org.jsoup.nodes.Document; +import org.junit.jupiter.api.*; + +import java.io.IOException; +import java.net.URL; + +public class RedgifsRipperTest extends RippersTest { + + /** + * Rips correctly formatted URL directly from Redgifs + * @throws IOException + */ + @Test + public void testRedgifsGoodURL() throws IOException{ + RedgifsRipper ripper = new RedgifsRipper(new URL("https://redgifs.com/watch/blaringbonyfulmar-panty-peel")); + testRipper(ripper); + } + + + /** + * Rips a Redifs profile + * @throws IOException + */ + @Test + public void testRedgifsProfile() throws IOException { + RedgifsRipper ripper = new RedgifsRipper(new URL("https://redgifs.com/users/margo_monty")); + testRipper(ripper); + } + + /** + * Rips a Redifs category/search + * @throws IOException + */ + @Test + public void testRedgifsSearch() throws IOException { + RedgifsRipper ripper = new RedgifsRipper(new URL("https://redgifs.com/gifs/browse/little-caprice")); + Document doc = ripper.getFirstPage(); + + doc = ripper.getNextPage(doc); + assertTrue("https://api.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=150".equalsIgnoreCase(doc.location())); + doc = ripper.getNextPage(doc); + assertTrue("https://api.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=300".equalsIgnoreCase(doc.location())); + } +} From 689968948cfbc5ef55f972996211a0a7a6261817 Mon Sep 17 00:00:00 2001 From: borderline232 Date: Sun, 7 Jun 2020 00:07:09 -0400 Subject: [PATCH 034/512] RedgifsRipper: Fixes to redgifs ripper - Changed redgifs api URL to reflect updated change - Added check in getNextPage to see if page has any URLs else finishes rather than throwing error - Added gifdeliverynetwork.com as knwon host as it redirects to redgifs page - Added reddit compatibility by adding as known host in RipUtils reddit ripper - Unit test to cover above changes --- .../ripme/ripper/rippers/RedgifsRipper.java | 43 +++++++++++++------ .../com/rarchives/ripme/utils/RipUtils.java | 13 ++++++ .../tst/ripper/rippers/RedgifsRipperTest.java | 16 +++++-- 3 files changed, 54 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java index 9d6152b9..17105ee4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java @@ -19,6 +19,7 @@ import java.util.regex.Pattern; public class RedgifsRipper extends AbstractHTMLRipper { private static final String HOST = "redgifs.com"; + private static final String HOST_2 = "gifdeliverynetwork.com"; String username = ""; String cursor = ""; String count = "100"; @@ -41,7 +42,7 @@ public class RedgifsRipper extends AbstractHTMLRipper { @Override public boolean canRip(URL url) { - return url.getHost().endsWith(HOST); + return url.getHost().endsWith(HOST) || url.getHost().endsWith(HOST_2); } @Override @@ -49,6 +50,7 @@ public class RedgifsRipper extends AbstractHTMLRipper { String sUrl = url.toExternalForm(); sUrl = sUrl.replace("/gifs/detail", ""); sUrl = sUrl.replace("/amp", ""); + sUrl = sUrl.replace("gifdeliverynetwork.com", "redgifs.com/watch"); return new URL(sUrl); } @@ -74,10 +76,10 @@ public class RedgifsRipper extends AbstractHTMLRipper { } else if (isSearch().matches()) { searchText = getGID(url).replace("-", " "); return Http.url( - new URL("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart*searchCount)).ignoreContentType().get(); + new URL("https://napi.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart*searchCount)).ignoreContentType().get(); } else { username = getGID(url); - return Http.url(new URL("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count)) + return Http.url(new URL("https://napi.redgifs.com/v1/users/" + username + "/gfycats?count=" + count)) .ignoreContentType().get(); } } @@ -122,16 +124,18 @@ public class RedgifsRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException { - if (!isProfile().matches()) { - return Http.url( - new URL("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + if (isSearch().matches()) { + Document d = Http.url( + new URL("https://napi.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchCount*++searchStart)) .ignoreContentType().get(); + return (hasURLs(d).isEmpty()) ? null : d; } else { if (cursor.equals("")) { - throw new IOException("No more pages"); + return null; } else { - return Http.url(new URL("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor)).ignoreContentType().get(); + Document d = Http.url(new URL("https://napi.redgifs.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor)).ignoreContentType().get(); + return (hasURLs(d).isEmpty()) ? null : d; } } } @@ -140,12 +144,7 @@ public class RedgifsRipper extends AbstractHTMLRipper { public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); if (isProfile().matches() || isSearch().matches()) { - JSONObject page = new JSONObject(stripHTMLTags(doc.html())); - JSONArray content = page.getJSONArray("gfycats"); - for (int i = 0; i < content.length(); i++) { - result.add(content.getJSONObject(i).getString("mp4Url")); - } - cursor = page.getString("cursor"); + result = hasURLs(doc); } else { Elements videos = doc.select("script"); for (Element el : videos) { @@ -159,6 +158,22 @@ public class RedgifsRipper extends AbstractHTMLRipper { return result; } + /** + * Helper method for retrieving URLs. + * @param doc Document of the URL page to look through + * @return List of URLs to download + */ + public List hasURLs(Document doc) { + List result = new ArrayList<>(); + JSONObject page = new JSONObject(stripHTMLTags(doc.html())); + JSONArray content = page.getJSONArray("gfycats"); + for (int i = 0; i < content.length(); i++) { + result.add(content.getJSONObject(i).getString("mp4Url")); + } + cursor = page.getString("cursor"); + return result; + } + /** * Helper method for retrieving video URLs. * @param url URL to gfycat page diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 03a480cf..3fcb71c2 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -11,6 +11,7 @@ import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.ripper.rippers.EroShareRipper; import com.rarchives.ripme.ripper.rippers.EromeRipper; import com.rarchives.ripme.ripper.rippers.ImgurRipper; +import com.rarchives.ripme.ripper.rippers.RedgifsRipper; import com.rarchives.ripme.ripper.rippers.VidbleRipper; import com.rarchives.ripme.ripper.rippers.GfycatRipper; import org.apache.commons.lang.math.NumberUtils; @@ -76,6 +77,18 @@ public class RipUtils { } return result; } + else if (url.getHost().endsWith("redgifs.com") || url.getHost().endsWith("gifdeliverynetwork.com")) { + try { + logger.debug("Fetching redgifs page " + url); + String videoURL = RedgifsRipper.getVideoURL(url); + logger.debug("Got redgifs URL: " + videoURL); + result.add(new URL(videoURL)); + } catch (IOException e) { + // Do nothing + logger.warn("Exception while retrieving redgifs page:", e); + } + return result; + } else if (url.toExternalForm().contains("vidble.com/album/") || url.toExternalForm().contains("vidble.com/show/")) { try { logger.info("Getting vidble album " + url); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java index 46624303..922dbaf8 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java @@ -1,6 +1,5 @@ package com.rarchives.ripme.tst.ripper.rippers; -import com.rarchives.ripme.ripper.rippers.GfycatRipper; import com.rarchives.ripme.ripper.rippers.RedgifsRipper; import org.jsoup.nodes.Document; import org.junit.jupiter.api.*; @@ -16,10 +15,19 @@ public class RedgifsRipperTest extends RippersTest { */ @Test public void testRedgifsGoodURL() throws IOException{ - RedgifsRipper ripper = new RedgifsRipper(new URL("https://redgifs.com/watch/blaringbonyfulmar-panty-peel")); + RedgifsRipper ripper = new RedgifsRipper(new URL("https://www.redgifs.com/watch/talkativewarpeddragon-petite")); testRipper(ripper); } + /** + * Rips gifdeliverynetwork URL's by redirecting them to proper redgifs url + * @throws IOException + */ + @Test + public void testRedgifsBadRL() throws IOException{ + RedgifsRipper ripper = new RedgifsRipper(new URL("https://www.gifdeliverynetwork.com/foolishelasticchimpanzee")); + testRipper(ripper); + } /** * Rips a Redifs profile @@ -41,8 +49,8 @@ public class RedgifsRipperTest extends RippersTest { Document doc = ripper.getFirstPage(); doc = ripper.getNextPage(doc); - assertTrue("https://api.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=150".equalsIgnoreCase(doc.location())); + assertTrue("https://napi.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=150".equalsIgnoreCase(doc.location())); doc = ripper.getNextPage(doc); - assertTrue("https://api.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=300".equalsIgnoreCase(doc.location())); + assertTrue("https://napi.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=300".equalsIgnoreCase(doc.location())); } } From 2848815bd7d74deddc1a0ee115aadfa0023f0b80 Mon Sep 17 00:00:00 2001 From: RAISSA Date: Fri, 12 Jun 2020 22:58:33 -0400 Subject: [PATCH 035/512] Adding missing translation to Korean/KR --- .../resources/LabelsBundle_kr_KR.properties | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/main/resources/LabelsBundle_kr_KR.properties b/src/main/resources/LabelsBundle_kr_KR.properties index 34a35a7a..e0110055 100644 --- a/src/main/resources/LabelsBundle_kr_KR.properties +++ b/src/main/resources/LabelsBundle_kr_KR.properties @@ -35,23 +35,23 @@ inactive = \uBE44\uD65C\uC131\uD654 re-rip.checked = \uB2E4\uC2DC \uBCF5\uC0AC\uD558\uAE30 \uCCB4\uD06C\uB428 remove = \uC120\uD0DD\uD55C \uAE30\uB85D \uC0AD\uC81C clear = \uD788\uC2A4\uD1A0\uB9AC \uBAA8\uB450 \uC0AD\uC81C -download.url.list = Download url list -select.save.dir = Select Save Directory +download.url.list = URL \ubAA9\uB85D\uC744 \uB2E4\uC6B4\uB85C\uB4DC\uD558\uAE30 +select.save.dir = \uC800\uC7A5\uC744 \uB514\uB809\uD1A0\uB9AC \uC120\uD0DD\uD558\uAE30 # Keys for the logs generated by DownloadFileThread -nonretriable.status.code = Non-retriable status code -retriable.status.code = Retriable status code -server.doesnt.support.resuming.downloads = Server doesn't support resuming downloads +nonretriable.status.code = \uBCF5\uAD6C\uD560 \uC218 \uC5C6\uB294 \uC0C1\uD0DC \uCF54\uB4DC +retriable.status.code = \uBCF5\uAD6C\uD560 \uC218 \uC788\uB294 \uC0C1\uD0DC \uCF54\uB4DC +server.doesnt.support.resuming.downloads = \uC11C\uBC84 \uB2E4\uC6B4\uB85C\uB4DC \uC7AC\uAC1C\uB97C \uC9C0\uC6D0\uD558\uC9C0 \uC54A\uB2E4 # A "magic number" can also be called a file signature -was.unable.to.get.content.type.using.magic.number = Was unable to get content type using magic number -magic.number.was = Magic number was -deleting.existing.file = Deleting existing file -request.properties = Request properties -download.interrupted = Download interrupted -exceeded.maximum.retries = Exceeded maximum retries -http.status.exception = HTTP status exception -exception.while.downloading.file = Exception while downloading file -failed.to.download = Failed to download -skipping = Skipping -file.already.exists = file already exists \ No newline at end of file +was.unable.to.get.content.type.using.magic.number = \uC2DC\uADF8\uB108\uCC98 \uD30C\uC77C \uC0AC\uC6A9\uD558\uC5EC \uB0B4\uC6A9 \uC720\uD615\uC744 \uAC00\uC838\uC62C \uC218 \uC5C6\uAE30 +magic.number.was = \uC2DC\uADF8\uB108\uCC98 \uD30C\uC77C \uC774\uAC70\uC600\uC2B5\uB2C8\uB2E4 +deleting.existing.file = \uAE30\uC874 \uD30C\uC77C\uC744 \uC0AD\uC81C\uD558\uB294 \uC911\uC785\uB2C8\uB2E4 +request.properties = \uC18D\uC131\uC744 \uC694\uCCAD\uD558\uAE30 +download.interrupted = \uB2E4\uC6B4\uB85C\uB4DC\uAC00 \uC911\uB2E8\uB418\uC5C8\uC2B5\uB2C8\uB2E4 +exceeded.maximum.retries = \uCD5C\uB300 \uC7AC\uC2DC\uB3C4 \uD69F\uC218\uB97C \uCD08\uACFC\uD588\uC2B5\uB2C8\uB2E4 +http.status.exception = HTTP \uC0C1\uD0DC \uC608\uC678\uB97C +exception.while.downloading.file = \uD30C\uC77C\uC744 \uB2E4\uC6B4\uB85C\uB4DC\uD558\uB294 \uB3D9\uC548 \uC608\uC678\uAC00 \uBC1C\uC0DD\uD558\uAE30 +failed.to.download = \uB2E4\uC6B4\uB85C\uB4DC\uD558\uC9C0 \uBABB\uD558\uAE30 +skipping = \uAC74\uB108\uB6F0\uACE0 \uC788\uC2B5\uB2C8\uB2E4 +file.already.exists = \uD30C\uC77C\uC774 \uC774\uBBF8 \uC788\uC2B5\uB2C8\uB2E4 \ No newline at end of file From 4f43f1e2384751a146c3c591389f544d79f063b7 Mon Sep 17 00:00:00 2001 From: Ruthalas Date: Sat, 27 Jun 2020 22:15:04 -0700 Subject: [PATCH 036/512] Resolve xlecx error xlecx has changed its URL from 'xlecx.com' to 'xlecx.org'. This change modifies the ripper.java to reflect that change, and to add the required https prefix. If the https prefix is not added, the image download fails with 'no protocol'. That seems a little janky, but it was used in #120. --- .../com/rarchives/ripme/ripper/rippers/XcartxRipper.java | 2 +- .../com/rarchives/ripme/ripper/rippers/XlecxRipper.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XcartxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XcartxRipper.java index 5ed01e6f..64829a0b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XcartxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XcartxRipper.java @@ -56,7 +56,7 @@ public class XcartxRipper extends AbstractHTMLRipper { for (Element image : imageElements) { String imageUrl = image.attr("data-src"); - imageURLs.add(getDomain() + imageUrl); + imageURLs.add("https://" + getDomain() + imageUrl); } return imageURLs; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XlecxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XlecxRipper.java index 15aee9c9..0aaacfc4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XlecxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XlecxRipper.java @@ -8,7 +8,7 @@ import java.util.regex.Pattern; public class XlecxRipper extends XcartxRipper { - private Pattern p = Pattern.compile("^https?://xlecx.com/([a-zA-Z0-9_\\-]+).html"); + private Pattern p = Pattern.compile("^https?://xlecx.org/([a-zA-Z0-9_\\-]+).html"); public XlecxRipper(URL url) throws IOException { super(url); @@ -21,7 +21,7 @@ public class XlecxRipper extends XcartxRipper { @Override public String getDomain() { - return "xlecx.com"; + return "xlecx.org"; } @Override @@ -30,7 +30,7 @@ public class XlecxRipper extends XcartxRipper { if (m.matches()) { return m.group(1); } - throw new MalformedURLException("Expected URL format: http://xlecx.com/comic, got: " + url); + throw new MalformedURLException("Expected URL format: http://xlecx.org/comic, got: " + url); } } From 2e221e571339243356dfb6fdf5bcbd0de84966d3 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 26 Jul 2020 11:34:20 +0200 Subject: [PATCH 037/512] com.rarchives.ripme.tst from junit3 to junit5 AppTest is not migrated as it only does true=true, otherwise it is not clear what this should be. --- .../java/com/rarchives/ripme/tst/AbstractRipperTest.java | 6 ++++-- src/test/java/com/rarchives/ripme/tst/Base64Test.java | 7 +++++-- src/test/java/com/rarchives/ripme/tst/proxyTest.java | 7 +++++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java b/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java index f1d8eff5..a388151c 100644 --- a/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java @@ -1,15 +1,17 @@ package com.rarchives.ripme.tst; import com.rarchives.ripme.ripper.AbstractRipper; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.net.URL; +import static org.junit.jupiter.api.Assertions.assertEquals; -public class AbstractRipperTest extends TestCase { +public class AbstractRipperTest { + @Test public void testGetFileName() throws IOException { String fileName = AbstractRipper.getFileName(new URL("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D"), "test", "test"); assertEquals("test.test", fileName); diff --git a/src/test/java/com/rarchives/ripme/tst/Base64Test.java b/src/test/java/com/rarchives/ripme/tst/Base64Test.java index ffe2f3f8..f5c82078 100644 --- a/src/test/java/com/rarchives/ripme/tst/Base64Test.java +++ b/src/test/java/com/rarchives/ripme/tst/Base64Test.java @@ -1,10 +1,13 @@ package com.rarchives.ripme.tst; -import junit.framework.TestCase; import com.rarchives.ripme.utils.Base64; +import org.junit.jupiter.api.Test; -public class Base64Test extends TestCase { +import static org.junit.jupiter.api.Assertions.assertEquals; +public class Base64Test { + + @Test public void testDecode() { assertEquals("test", new String(Base64.decode("dGVzdA=="))); } diff --git a/src/test/java/com/rarchives/ripme/tst/proxyTest.java b/src/test/java/com/rarchives/ripme/tst/proxyTest.java index 36ea2f55..72140853 100644 --- a/src/test/java/com/rarchives/ripme/tst/proxyTest.java +++ b/src/test/java/com/rarchives/ripme/tst/proxyTest.java @@ -4,14 +4,16 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.utils.Proxy; import com.rarchives.ripme.utils.Utils; -import junit.framework.TestCase; import com.rarchives.ripme.utils.Http; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertFalse; -public class proxyTest extends TestCase { +public class proxyTest { // This test will only run on machines where the user has added a entry for proxy.socks + @Test public void testSocksProxy() throws IOException { // Unset proxy before testing System.setProperty("http.proxyHost", ""); @@ -31,6 +33,7 @@ public class proxyTest extends TestCase { } // This test will only run on machines where the user has added a entry for proxy.http + @Test public void testHTTPProxy() throws IOException { // Unset proxy before testing System.setProperty("http.proxyHost", ""); From ed0dfc878c7e0c2f8a2d8e07116641bc096e5913 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 26 Jul 2020 11:37:26 +0200 Subject: [PATCH 038/512] delete AppTest which is only true=true --- .../java/com/rarchives/ripme/tst/AppTest.java | 30 ------------------- 1 file changed, 30 deletions(-) delete mode 100644 src/test/java/com/rarchives/ripme/tst/AppTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/AppTest.java b/src/test/java/com/rarchives/ripme/tst/AppTest.java deleted file mode 100644 index fb9bc680..00000000 --- a/src/test/java/com/rarchives/ripme/tst/AppTest.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.rarchives.ripme.tst; - -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; - -public class AppTest extends TestCase { - /** - * Create the test case - * - * @param testName name of the test case - */ - public AppTest(String testName) { - super(testName); - } - - /** - * @return the suite of tests being tested - */ - public static Test suite() { - return new TestSuite(AppTest.class); - } - - /** - * Rigourous Test :-) - */ - public void testApp() { - assertTrue(true); - } -} From 8662dd3c44fb1944988733e4aa16bc1dcfc5b89b Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 26 Jul 2020 13:50:36 +0200 Subject: [PATCH 039/512] upgrade to junit-bom-5.6.2 --- pom.xml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 4a876b98..3d2cdf11 100644 --- a/pom.xml +++ b/pom.xml @@ -9,14 +9,22 @@ UTF-8 4.12 - 5.5.0 - 5.5.0 + + + + org.junit + junit-bom + 5.6.2 + pom + import + + + org.junit.jupiter junit-jupiter-api - ${junit.jupiter.version} test @@ -28,13 +36,11 @@ org.junit.jupiter junit-jupiter-engine - ${junit.jupiter.version} test org.junit.vintage junit-vintage-engine - ${junit.vintage.version} test From 298c2c2ec37d0a8748a07bb66f78ceddeb9f1c32 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 26 Jul 2020 14:05:42 +0200 Subject: [PATCH 040/512] no junit4 dependency necessary any more --- pom.xml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pom.xml b/pom.xml index 3d2cdf11..977176f2 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,6 @@ http://rip.rarchives.com UTF-8 - 4.12 @@ -27,12 +26,6 @@ junit-jupiter-api test - - junit - junit - ${junit.version} - test - org.junit.jupiter junit-jupiter-engine From df431bef2b21f136d502fe593d8e450ef2e5b328 Mon Sep 17 00:00:00 2001 From: Kleptoc Date: Tue, 28 Jul 2020 13:36:13 -0700 Subject: [PATCH 041/512] Added ripper for HentaiNexus --- .../ripper/rippers/HentaiNexusRipper.java | 129 ++++++++++++++++++ .../ripper/rippers/HentainexusRipperTest.java | 15 ++ 2 files changed, 144 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java new file mode 100644 index 00000000..56ce0d2f --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java @@ -0,0 +1,129 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.ripper.DownloadThreadPool; +import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.Utils; + +public class HentaiNexusRipper extends AbstractHTMLRipper { + + private Document firstPage; + private DownloadThreadPool hentainexusThreadPool = new DownloadThreadPool("hentainexus"); + @Override + public DownloadThreadPool getThreadPool() { + return hentainexusThreadPool; + } + + public HentaiNexusRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "hentainexus"; + } + + @Override + public String getDomain() { + return "hentainexus.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https?://hentainexus\\.com/view/([a-zA-Z0-9_\\-%]*)/?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected hentainexus.com URL format: " + + "hentainexus.com/view/NUMBER - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + if (firstPage == null) { + firstPage = Http.url(url).get(); + } + return firstPage; + } + + @Override + public List getURLsFromPage(Document doc) { + List imageURLs = new ArrayList<>(); + Elements thumbs = doc.select("div.is-multiline > div.column > a"); + for (Element el : thumbs) { + imageURLs.add("https://" + getDomain() + el.attr("href")); + } + return imageURLs; + } + + @Override + public String getAlbumTitle(URL url) throws MalformedURLException { + try { + Document gallery = Http.url(url).get(); + return getHost() + "_" + gallery.select("h1.title").text(); + } catch (IOException e) { + LOGGER.info("Falling back"); + } + + return super.getAlbumTitle(url); + } + + @Override + public void downloadURL(URL url, int index) { + HentaiNexusImageThread t = new HentaiNexusImageThread(url, index); + hentainexusThreadPool.addThread(t); + } + + /** + * Helper class to find and download images found on "image" pages + */ + private class HentaiNexusImageThread extends Thread { + private URL url; + private int index; + + HentaiNexusImageThread(URL url, int index) { + super(); + this.url = url; + this.index = index; + } + + @Override + public void run() { + fetchImage(); + } + + private void fetchImage() { + try { + Document doc = Http.url(url).retries(3).get(); + Elements images = doc.select("figure.image > img"); + if (images.isEmpty()) { + LOGGER.warn("Image not found at " + this.url); + return; + } + Element image = images.first(); + String imgsrc = image.attr("src"); + String prefix = ""; + if (Utils.getConfigBoolean("download.save_order", true)) { + prefix = String.format("%03d_", index); + } + addURLToDownload(new URL(imgsrc), prefix); + } catch (IOException e) { + LOGGER.error("[!] Exception while loading/parsing " + this.url, e); + } + } + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java new file mode 100644 index 00000000..cfe540fb --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java @@ -0,0 +1,15 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.HentaiNexusRipper; +import org.junit.jupiter.api.Test; + +public class HentainexusRipperTest extends RippersTest { + @Test + public void testHentaiNexusAlbum() throws IOException { + HentaiNexusRipper ripper = new HentaiNexusRipper(new URL("https://hentainexus.com/view/44")); + testRipper(ripper); + } +} From 4dbffc660e2b228a59f58343009bb76c505d64ae Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 29 Jul 2020 05:23:46 +0200 Subject: [PATCH 042/512] junit5 VscoRipper test + fix it --- .../ripme/tst/ripper/rippers/RippersTest.java | 5 ----- .../tst/ripper/rippers/VscoRipperTest.java | 20 +++++-------------- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java index 87530881..6677c02c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java @@ -131,11 +131,6 @@ public class RippersTest { Assertions.assertEquals(expected, actual); } - @Deprecated - void assertEquals(String message, String expected, String actual) { - Assertions.assertEquals(expected, actual, message); - } - @Deprecated void assertEquals(Object expected, Object actual) { Assertions.assertEquals(expected, actual); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java index 3b3bdaa3..fc78ec2d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.VscoRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -21,25 +22,14 @@ public class VscoRipperTest extends RippersTest { } /** - * Tests profile rip. - * - * @throws IOException - */ - @Test - public void testProfileRip() throws IOException { - VscoRipper ripper = new VscoRipper(new URL("https://vsco.co/jonathangodoy/images/1")); - testRipper(ripper); - } - - /** - * Prevents Bug #679 from happening again. + * Tests profile rip., Prevents Bug #679 from happening again. * https://github.com/RipMeApp/ripme/issues/679 * * @throws IOException */ @Test public void testHyphenatedRip() throws IOException { - VscoRipper ripper = new VscoRipper(new URL("https://vsco.co/jolly-roger/images/1")); + VscoRipper ripper = new VscoRipper(new URL("https://vsco.co/jolly-roger/gallery")); testRipper(ripper); } @@ -50,11 +40,11 @@ public class VscoRipperTest extends RippersTest { */ @Test public void testGetGID() throws IOException { - URL url = new URL("https://vsco.co/minijello/media/571cd612542220261a123441"); + URL url = new URL("https://vsco.co/jolly-roger/media/590359c4ade3041f2658f407"); VscoRipper ripper = new VscoRipper(url); - assertEquals("Failed to get GID", "minijello/571cd", ripper.getGID(url)); + Assertions.assertEquals("jolly-roger/59035", ripper.getGID(url), "Failed to get GID"); } } From 4d85d72e68b39c426fde1a62e2dfa4f3023c3db4 Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 29 Jul 2020 06:10:20 +0200 Subject: [PATCH 043/512] newsfilter domain does not exist any more --- .../ripper/rippers/NewsfilterRipper.java | 80 ------------------- .../ripper/rippers/NewsfilterRipperTest.java | 15 ---- 2 files changed, 95 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/NewsfilterRipper.java delete mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/NewsfilterRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NewsfilterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NewsfilterRipper.java deleted file mode 100644 index bafa3690..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NewsfilterRipper.java +++ /dev/null @@ -1,80 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -public class NewsfilterRipper extends AbstractHTMLRipper { - - private static final String HOST = "newsfilter"; - private static final String DOMAIN = "newsfilter.org"; - - public NewsfilterRipper(URL url) throws IOException { - super(url); - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - String u = url.toExternalForm(); - if (u.indexOf('#') >= 0) { - u = u.substring(0, u.indexOf('#')); - } - u = u.replace("https?://m\\.newsfilter\\.org", "http://newsfilter.org"); - return new URL(u); - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://([wm]+\\.)?newsfilter\\.org/gallery/([^/]+)$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(2); - } - throw new MalformedURLException( - "Expected newsfilter gallery format: http://newsfilter.org/gallery/galleryid" + - " Got: " + url); - } - - @Override - public String getHost() { - return HOST; - } - - @Override - protected String getDomain() { - return DOMAIN; - } - - @Override - protected Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - - @Override - protected List getURLsFromPage(Document page) { - List imgURLs = new ArrayList<>(); - Elements thumbnails = page.select("#galleryImages .inner-block img"); - for (Element thumb : thumbnails) { - String thumbUrl = thumb.attr("src"); - String picUrl = thumbUrl.replace("thumbs/", ""); - // use HTTP instead of HTTPS (less headaches) - imgURLs.add(picUrl.replaceFirst("https://", "http://")); - } - return imgURLs; - } - - @Override - protected void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } -} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NewsfilterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NewsfilterRipperTest.java deleted file mode 100644 index 8567b0ff..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NewsfilterRipperTest.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URL; - -import com.rarchives.ripme.ripper.rippers.NewsfilterRipper; -import org.junit.jupiter.api.Test; - -public class NewsfilterRipperTest extends RippersTest { - @Test - public void testNewsfilterRip() throws IOException { - NewsfilterRipper ripper = new NewsfilterRipper(new URL("http://newsfilter.org/gallery/he-doubted-she-would-fuck-on-cam-happy-to-be-proven-wrong-216799")); - testRipper(ripper); - } -} \ No newline at end of file From bd72bef3f778d204f3e6c38e10f12a4ebf1c80f5 Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 29 Jul 2020 06:47:00 +0200 Subject: [PATCH 044/512] tubex6 domain does not exist any more --- .../ripme/ripper/rippers/Tubex6Ripper.java | 60 ------------------- .../tst/ripper/rippers/Tubex6RipperTest.java | 19 ------ 2 files changed, 79 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/Tubex6Ripper.java delete mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/Tubex6RipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Tubex6Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Tubex6Ripper.java deleted file mode 100644 index 35ca8281..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Tubex6Ripper.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.rarchives.ripme.ripper.AbstractSingleFileRipper; -import org.jsoup.nodes.Document; - -import com.rarchives.ripme.utils.Http; - -public class Tubex6Ripper extends AbstractSingleFileRipper { - - public Tubex6Ripper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "tubex6"; - } - - @Override - public String getDomain() { - return "tubex6.com"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^http://.*tubex6\\.com/(.*)/$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - throw new MalformedURLException("Expected tubex6.com URL format: " + - "tubex6.com/NAME - got " + url + " instead"); - } - - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - - @Override - public List getURLsFromPage(Document doc) { - List result = new ArrayList<>(); - result.add(doc.select("source[type=video/mp4]").attr("src")); - return result; - } - - @Override - public void downloadURL(URL url, int index) { - // We have to send a referrer or the site returns a 403 error - addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), null); - } -} \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Tubex6RipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Tubex6RipperTest.java deleted file mode 100644 index 83ff88a1..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Tubex6RipperTest.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URL; - -import com.rarchives.ripme.ripper.rippers.Tubex6Ripper; - -public class Tubex6RipperTest extends RippersTest { - public void testRip() throws IOException { - Tubex6Ripper ripper = new Tubex6Ripper(new URL("http://www.tubex6.com/my-sister-sleeps-naked-1/")); - testRipper(ripper); - } - - public void testGetGID() throws IOException { - URL url = new URL("http://www.tubex6.com/my-sister-sleeps-naked-1/"); - Tubex6Ripper ripper = new Tubex6Ripper(url); - assertEquals("my-sister-sleeps-naked-1", ripper.getGID(url)); - } -} \ No newline at end of file From b1e1cfbe8c9a94d8146abc7d0f2333a1c9ca3c48 Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 29 Jul 2020 06:57:49 +0200 Subject: [PATCH 045/512] broker ripper test switch off --- .../ripme/tst/ripper/rippers/TsuminoRipperTest.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TsuminoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TsuminoRipperTest.java index b1e58adc..38dee451 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TsuminoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TsuminoRipperTest.java @@ -7,33 +7,37 @@ import java.util.List; import com.rarchives.ripme.ripper.rippers.TsuminoRipper; import com.rarchives.ripme.utils.RipUtils; import org.jsoup.nodes.Document; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class TsuminoRipperTest extends RippersTest { @Test + @Disabled("Broken ripper") public void testTsuminoRipper() throws IOException { TsuminoRipper ripper = new TsuminoRipper(new URL("http://www.tsumino.com/Book/Info/43528/sore-wa-kurokute-suketeita-what-s-tight-and-black-and-sheer-all-over-")); testRipper(ripper); } @Test + @Disabled("Broken ripper") public void testTagBlackList() throws IOException { TsuminoRipper ripper = new TsuminoRipper(new URL("http://www.tsumino.com/Book/Info/43528/sore-wa-kurokute-suketeita-what-s-tight-and-black-and-sheer-all-over-")); Document doc = ripper.getFirstPage(); List tagsOnPage = ripper.getTags(doc); String[] tags1 = {"test", "one", "Smell"}; String blacklistedTag = RipUtils.checkTags(tags1, tagsOnPage); - assertEquals("smell", blacklistedTag); + Assertions.assertEquals("smell", blacklistedTag); // Test a tag with spaces String[] tags2 = {"test", "one", "Face sitting"}; blacklistedTag = RipUtils.checkTags(tags2, tagsOnPage); - assertEquals("face sitting", blacklistedTag); + Assertions.assertEquals("face sitting", blacklistedTag); // Test a album with no blacklisted tags String[] tags3 = {"nothing", "one", "null"}; blacklistedTag = RipUtils.checkTags(tags3, tagsOnPage); - assertNull(blacklistedTag); + Assertions.assertNull(blacklistedTag); } } \ No newline at end of file From e242515a4aa6c3207993862ead0d3ec5cf536b88 Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 29 Jul 2020 07:10:19 +0200 Subject: [PATCH 046/512] sinnercomis domain does not exist --- .../ripper/rippers/SinnercomicsRipper.java | 168 ------------------ .../rippers/SinnercomicsRipperTest.java | 33 ---- 2 files changed, 201 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/SinnercomicsRipper.java delete mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinnercomicsRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SinnercomicsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SinnercomicsRipper.java deleted file mode 100644 index 4411adfe..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SinnercomicsRipper.java +++ /dev/null @@ -1,168 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -public class SinnercomicsRipper extends AbstractHTMLRipper { - - private static final String HOST = "sinnercomics", - DOMAIN = "sinnercomics.com"; - - private static final int SLEEP_TIME = 500; - - enum RIP_TYPE { - HOMEPAGE, - PINUP, - COMIC - } - - private RIP_TYPE ripType; - private Integer pageNum; - - public SinnercomicsRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return HOST; - } - - @Override - public String getDomain() { - return DOMAIN; - } - - @Override - public String normalizeUrl(String url) { - // Remove the comments hashtag - return url.replaceAll("/#(comments|disqus_thread)", "/"); - } - - @Override - public String getGID(URL url) throws MalformedURLException { - String cleanUrl = normalizeUrl(url.toExternalForm()); - Pattern p; - Matcher m; - - p = Pattern.compile("^https?://sinnercomics\\.com/comic/([a-zA-Z0-9-]*)/?$"); - m = p.matcher(cleanUrl); - if (m.matches()) { - // Comic - this.ripType = RIP_TYPE.COMIC; - return m.group(1).replaceAll("-page-\\d+", ""); - } - - p = Pattern.compile("^https?://sinnercomics\\.com(?:/page/([0-9]+))?/?$"); - m = p.matcher(cleanUrl); - if (m.matches()) { - // Homepage - this.ripType = RIP_TYPE.HOMEPAGE; - if (m.group(1) != null) { - this.pageNum = Integer.valueOf(m.group(1)); - } else { - this.pageNum = 1; - } - return "homepage"; - } - - p = Pattern.compile("^https?://sinnercomics\\.com/([a-zA-Z0-9-]+)(?:/#comments)?/?$"); - m = p.matcher(cleanUrl); - if (m.matches()) { - // Pinup image - this.ripType = RIP_TYPE.PINUP; - return m.group(1); - } - - throw new MalformedURLException("Expected sinnercomics.com URL format: " + - "/pinupName or /comic/albumName or /page/number - got " + cleanUrl + " instead"); - } - - @Override - public boolean canRip(URL url) { - if (!url.getHost().endsWith(DOMAIN)) { - return false; - } - try { - getGID(url); - } catch (MalformedURLException e) { - // Can't get GID, can't rip it. - return false; - } - return true; - } - - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - - @Override - public Document getNextPage(Document doc) throws IOException { - String nextUrl = null; - - switch (this.ripType) { - case PINUP: - throw new IOException("No next page on a pinup"); - - case COMIC: - // We use comic-nav-next to the find the next page - Element elem = doc.select("a.comic-nav-next").first(); - if (elem == null) { - throw new IOException("No more pages"); - } - nextUrl = elem.attr("href"); - break; - - default: // case HOMEPAGE: - this.pageNum++; - nextUrl = "https://sinnercomics.com/page/" + String.valueOf(this.pageNum); - break; - } - - // Wait to avoid IP bans - sleep(SLEEP_TIME); - return Http.url(nextUrl).get(); - } - - @Override - public List getURLsFromPage(Document doc) { - List result = new ArrayList<>(); - - switch (this.ripType) { - case COMIC: - // comic pages only contain one image, determined by a meta tag - for (Element el : doc.select("meta[property=og:image]")) { - String imageSource = el.attr("content"); - imageSource = imageSource.replace(" alt=", ""); - result.add(imageSource); - } - break; - default: - for (Element el : doc.select(".entry p img")) { - // These filters match the full size images but might match ads too... - result.add(el.attr("src")); - } - break; - } - - return result; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } - -} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinnercomicsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinnercomicsRipperTest.java deleted file mode 100644 index 29ad8cf3..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinnercomicsRipperTest.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URL; - -import com.rarchives.ripme.ripper.rippers.SinnercomicsRipper; -import org.junit.jupiter.api.Test; - -public class SinnercomicsRipperTest extends RippersTest { - @Test - public void testSinnercomicsAlbum() throws IOException { - SinnercomicsRipper ripper; - - ripper = new SinnercomicsRipper(new URL("https://sinnercomics.com/comic/gw-addendum-page-01/")); - testRipper(ripper); - - } - - public void testGetGID() throws IOException { - URL url; - SinnercomicsRipper ripper; - - // Comic test - url = new URL("https://sinnercomics.com/comic/beyond-the-hotel-page-01/"); - ripper = new SinnercomicsRipper(url); - assertEquals("beyond-the-hotel", ripper.getGID(url)); - - // Comic test - url = new URL("https://sinnercomics.com/elza-frozen-2/#comments"); - ripper = new SinnercomicsRipper(url); - assertEquals("elza-frozen-2", ripper.getGID(url)); - } -} \ No newline at end of file From 084738c1f73cac1f29f0f732de5d00a340706e74 Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 29 Jul 2020 07:35:32 +0200 Subject: [PATCH 047/512] remove deprecated assertEquals --- .../tst/ripper/rippers/AerisdiesRipperTest.java | 3 ++- .../ripme/tst/ripper/rippers/BatoRipperTest.java | 5 +++-- .../tst/ripper/rippers/BooruRipperTest.java | 6 +++++- .../ripme/tst/ripper/rippers/ChanRipperTest.java | 14 +++++++------- .../tst/ripper/rippers/DeviantartRipperTest.java | 5 +++-- .../ripper/rippers/DynastyscansRipperTest.java | 6 +++++- .../ripme/tst/ripper/rippers/E621RipperTest.java | 5 +++-- .../tst/ripper/rippers/EhentaiRipperTest.java | 5 +++-- .../tst/ripper/rippers/EightmusesRipperTest.java | 5 +++-- .../tst/ripper/rippers/LusciousRipperTest.java | 6 +++--- .../tst/ripper/rippers/ManganeloRipperTest.java | 8 +++++++- .../tst/ripper/rippers/MeituriRipperTest.java | 3 ++- .../ripper/rippers/ModelmayhemRipperTest.java | 3 ++- .../ripper/rippers/MyhentaicomicsRipperTest.java | 13 +++++++------ .../rippers/MyhentaigalleryRipperTest.java | 4 +++- .../tst/ripper/rippers/NewgroundsRipperTest.java | 4 +++- .../tst/ripper/rippers/NfsfwRipperTest.java | 9 +++++---- .../tst/ripper/rippers/NhentaiRipperTest.java | 7 ++++--- .../tst/ripper/rippers/NudeGalsRipperTest.java | 6 +++++- .../tst/ripper/rippers/PornhubRipperTest.java | 16 ++++++++-------- .../ripme/tst/ripper/rippers/RippersTest.java | 5 ----- .../tst/ripper/rippers/Rule34RipperTest.java | 4 +++- .../tst/ripper/rippers/RulePornRipperTest.java | 10 +++++++--- .../ripper/rippers/SankakuComplexRipperTest.java | 5 +++-- .../tst/ripper/rippers/ShesFreakyRipperTest.java | 3 ++- .../tst/ripper/rippers/SinfestRipperTest.java | 6 +++++- .../tst/ripper/rippers/SmuttyRipperTest.java | 6 +++++- .../ripme/tst/ripper/rippers/StaRipperTest.java | 11 ++++++----- .../tst/ripper/rippers/TapasticRipperTest.java | 10 +++++++--- .../tst/ripper/rippers/TeenplanetRipperTest.java | 4 +++- .../ripper/rippers/TheyiffgalleryRipperTest.java | 4 +++- .../tst/ripper/rippers/VidbleRipperTest.java | 4 +++- .../ripme/tst/ripper/rippers/VkRipperTest.java | 5 +++-- .../tst/ripper/rippers/WebtoonsRipperTest.java | 11 +++++++---- .../ripper/rippers/WordpressComicRipperTest.java | 7 ++++--- .../tst/ripper/rippers/XhamsterRipperTest.java | 3 ++- .../tst/ripper/rippers/YuvutuRipperTest.java | 3 ++- .../tst/ripper/rippers/ZizkiRipperTest.java | 5 +++-- 38 files changed, 151 insertions(+), 88 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java index 5b99b8c4..d3166240 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java @@ -5,6 +5,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.AerisdiesRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class AerisdiesRipperTest extends RippersTest { @@ -30,7 +31,7 @@ public class AerisdiesRipperTest extends RippersTest { public void testGetGID() throws IOException { URL url = new URL("http://www.aerisdies.com/html/lb/douj_5230_1.html"); AerisdiesRipper ripper = new AerisdiesRipper(url); - assertEquals("5230", ripper.getGID(url)); + Assertions.assertEquals("5230", ripper.getGID(url)); } // TODO: Add a test for an album with a title. diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java index 635c7ecd..2ade8754 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java @@ -5,6 +5,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.BatoRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class BatoRipperTest extends RippersTest { @@ -19,13 +20,13 @@ public class BatoRipperTest extends RippersTest { public void testGetGID() throws IOException { URL url = new URL("https://bato.to/chapter/1207152"); BatoRipper ripper = new BatoRipper(url); - assertEquals("1207152", ripper.getGID(url)); + Assertions.assertEquals("1207152", ripper.getGID(url)); } @Test public void testGetAlbumTitle() throws IOException { URL url = new URL("https://bato.to/chapter/1207152"); BatoRipper ripper = new BatoRipper(url); - assertEquals("bato_1207152_I_Messed_Up_by_Teaching_at_a_Black_Gyaru_School!_Ch.2", ripper.getAlbumTitle(url)); + Assertions.assertEquals("bato_1207152_I_Messed_Up_by_Teaching_at_a_Black_Gyaru_School!_Ch.2", ripper.getAlbumTitle(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BooruRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BooruRipperTest.java index 31041175..8fa2cfc0 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BooruRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BooruRipperTest.java @@ -4,16 +4,20 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.BooruRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; public class BooruRipperTest extends RippersTest { + @Test public void testRip() throws IOException { BooruRipper ripper = new BooruRipper(new URL("http://xbooru.com/index.php?page=post&s=list&tags=furry")); testRipper(ripper); } + @Test public void testGetGID() throws IOException { URL url = new URL("http://xbooru.com/index.php?page=post&s=list&tags=furry"); BooruRipper ripper = new BooruRipper(url); - assertEquals("furry", ripper.getGID(url)); + Assertions.assertEquals("furry", ripper.getGID(url)); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java index 22995002..57c56fe3 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java @@ -1,6 +1,5 @@ package com.rarchives.ripme.tst.ripper.rippers; -import java.io.File; import java.io.IOException; import java.net.URL; import java.util.ArrayList; @@ -11,6 +10,7 @@ import com.rarchives.ripme.ripper.rippers.ChanRipper; import com.rarchives.ripme.ripper.rippers.ripperhelpers.ChanSite; import com.rarchives.ripme.utils.Http; import org.jsoup.nodes.Document; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class ChanRipperTest extends RippersTest { @@ -21,7 +21,7 @@ public class ChanRipperTest extends RippersTest { for (URL url : failURLs) { try { new ChanRipper(url); - fail("Instantiated ripper for URL that should not work: " + url); + Assertions.fail("Instantiated ripper for URL that should not work: " + url); } catch (Exception e) { // Expected } @@ -40,7 +40,7 @@ public class ChanRipperTest extends RippersTest { ChanRipper ripper = new ChanRipper(url); ripper.setup(); assert (ripper.canRip(url)); - assertNotNull("Ripper for " + url + " did not have a valid working directory.", ripper.getWorkingDir()); + Assertions.assertNotNull(ripper.getWorkingDir(), "Ripper for " + url + " did not have a valid working directory."); deleteDir(ripper.getWorkingDir()); } } @@ -54,11 +54,11 @@ public class ChanRipperTest extends RippersTest { ChanRipper ripper = new ChanRipper(new URL("http://desuchan.net/v/res/7034.html")); List chansFromConfig = ripper .getChansFromConfig("site1.com[cnd1.site1.com|cdn2.site2.biz],site2.co.uk[cdn.site2.co.uk]"); - assertEquals(chansFromConfig.get(0).getDomains(), site1); - assertEquals(chansFromConfig.get(0).getCdns(), site1Cdns); + Assertions.assertEquals(chansFromConfig.get(0).getDomains(), site1); + Assertions.assertEquals(chansFromConfig.get(0).getCdns(), site1Cdns); - assertEquals(chansFromConfig.get(1).getDomains(), site2); - assertEquals(chansFromConfig.get(1).getCdns(), site2Cdns); + Assertions.assertEquals(chansFromConfig.get(1).getDomains(), site2); + Assertions.assertEquals(chansFromConfig.get(1).getCdns(), site2Cdns); } @Test public void testChanRipper() throws IOException { diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java index adf15442..33003e5d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java @@ -8,6 +8,7 @@ import java.util.List; import com.rarchives.ripme.ripper.rippers.DeviantartRipper; import com.rarchives.ripme.utils.Http; import org.jsoup.nodes.Document; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -32,7 +33,7 @@ public class DeviantartRipperTest extends RippersTest { public void testGetGID() throws IOException { URL url = new URL("https://www.deviantart.com/airgee/gallery/"); DeviantartRipper ripper = new DeviantartRipper(url); - assertEquals("airgee", ripper.getGID(url)); + Assertions.assertEquals("airgee", ripper.getGID(url)); } @Test @@ -55,7 +56,7 @@ public class DeviantartRipperTest extends RippersTest { for (URL url : urls) { DeviantartRipper ripper = new DeviantartRipper(url); - assertEquals("https://www.deviantart.com/airgee/gallery/", ripper.sanitizeURL(url).toExternalForm()); + Assertions.assertEquals("https://www.deviantart.com/airgee/gallery/", ripper.sanitizeURL(url).toExternalForm()); } } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java index 8eb8d88f..4c8d6416 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java @@ -4,15 +4,19 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.DynastyscansRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; public class DynastyscansRipperTest extends RippersTest { + @Test public void testRip() throws IOException { DynastyscansRipper ripper = new DynastyscansRipper(new URL("https://dynasty-scans.com/chapters/under_one_roof_ch01")); testRipper(ripper); } + @Test public void testGetGID() throws IOException { DynastyscansRipper ripper = new DynastyscansRipper(new URL("https://dynasty-scans.com/chapters/under_one_roof_ch01")); - assertEquals("under_one_roof_ch01", ripper.getGID(new URL("https://dynasty-scans.com/chapters/under_one_roof_ch01"))); + Assertions.assertEquals("under_one_roof_ch01", ripper.getGID(new URL("https://dynasty-scans.com/chapters/under_one_roof_ch01"))); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java index 41a3e33e..63249423 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.E621Ripper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class E621RipperTest extends RippersTest { @@ -30,7 +31,7 @@ public class E621RipperTest extends RippersTest { try { noNextPageRipper.getNextPage(noNextPageRipper.getFirstPage()); } catch (IOException e) { - assertEquals(e.getMessage(), "No more pages."); + Assertions.assertEquals(e.getMessage(), "No more pages."); } } @Test @@ -57,7 +58,7 @@ public class E621RipperTest extends RippersTest { try { noNextPageRipper.getNextPage(noNextPageRipper.getFirstPage()); } catch (IOException e) { - assertEquals(e.getMessage(), "No more pages."); + Assertions.assertEquals(e.getMessage(), "No more pages."); } } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java index 78234951..021e892f 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java @@ -6,6 +6,7 @@ import java.util.List; import com.rarchives.ripme.ripper.rippers.EHentaiRipper; import com.rarchives.ripme.utils.RipUtils; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class EhentaiRipperTest extends RippersTest { @@ -24,11 +25,11 @@ public class EhentaiRipperTest extends RippersTest { // Test multiple blacklisted tags String[] tags = {"test", "one", "yuri"}; String blacklistedTag = RipUtils.checkTags(tags, tagsOnPage); - assertEquals("yuri", blacklistedTag); + Assertions.assertEquals("yuri", blacklistedTag); // test tags with spaces in them String[] tags2 = {"test", "one", "midnight on mars"}; blacklistedTag = RipUtils.checkTags(tags2, tagsOnPage); - assertEquals("midnight on mars", blacklistedTag); + Assertions.assertEquals("midnight on mars", blacklistedTag); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java index 2cdbcb2e..e3e9bcb7 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.EightmusesRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class EightmusesRipperTest extends RippersTest { @@ -22,11 +23,11 @@ public class EightmusesRipperTest extends RippersTest { @Test public void testGID() throws IOException { EightmusesRipper ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore")); - assertEquals("Affect3D-Comics", ripper.getGID(new URL("https://www.8muses.com/comics/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore"))); + Assertions.assertEquals("Affect3D-Comics", ripper.getGID(new URL("https://www.8muses.com/comics/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore"))); } @Test public void testGetSubdir() throws IOException { EightmusesRipper ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore")); - assertEquals("After-Party-Issue-1", ripper.getSubdir("After Party - Issue 1")); + Assertions.assertEquals("After-Party-Issue-1", ripper.getSubdir("After Party - Issue 1")); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java index c6febd32..52c4a3c5 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java @@ -4,7 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.LusciousRipper; -import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class LusciousRipperTest extends RippersTest { @@ -20,7 +20,7 @@ public class LusciousRipperTest extends RippersTest { public void testGetGID() throws IOException { URL url = new URL("https://luscious.net/albums/h-na-alice-wa-suki-desu-ka-do-you-like-alice-when_321609/"); LusciousRipper ripper = new LusciousRipper(url); - assertEquals("h-na-alice-wa-suki-desu-ka-do-you-like-alice-when_321609", ripper.getGID(url)); + Assertions.assertEquals("h-na-alice-wa-suki-desu-ka-do-you-like-alice-when_321609", ripper.getGID(url)); } @Test @@ -34,7 +34,7 @@ public class LusciousRipperTest extends RippersTest { try { singlePageRipper.getNextPage(singlePageRipper.getFirstPage()); } catch (IOException e) { - assertEquals("No next page found.", e.getMessage()); + Assertions.assertEquals("No next page found.", e.getMessage()); } } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java index 7eb1b8e9..37818121 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java @@ -4,16 +4,22 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ManganeloRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; public class ManganeloRipperTest extends RippersTest { + @Test + @Disabled("no images found, test or ripper broken") public void testRip() throws IOException { ManganeloRipper ripper = new ManganeloRipper(new URL("https://manganelo.com/manga/demonic_housekeeper")); testRipper(ripper); } + @Test public void testGetGID() throws IOException { URL url = new URL("https://manganelo.com/manga/demonic_housekeeper"); ManganeloRipper ripper = new ManganeloRipper(url); - assertEquals("demonic_housekeeper", ripper.getGID(url)); + Assertions.assertEquals("demonic_housekeeper", ripper.getGID(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MeituriRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MeituriRipperTest.java index fbd9ea57..883b73e3 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MeituriRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MeituriRipperTest.java @@ -5,6 +5,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.MeituriRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -20,6 +21,6 @@ public class MeituriRipperTest extends RippersTest { public void testGetGID() throws IOException { URL url = new URL("https://www.meituri.com/a/14449/"); MeituriRipper ripper = new MeituriRipper(url); - assertEquals("14449", ripper.getGID(url)); + Assertions.assertEquals("14449", ripper.getGID(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java index a9c859c7..9e81102a 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java @@ -5,6 +5,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.ModelmayhemRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -22,6 +23,6 @@ public class ModelmayhemRipperTest extends RippersTest { public void testGetGID() throws IOException { ModelmayhemRipper ripper = new ModelmayhemRipper( new URL("https://www.modelmayhem.com/portfolio/520206/viewall")); - assertEquals("520206", ripper.getGID(new URL("https://www.modelmayhem.com/portfolio/520206/viewall"))); + Assertions.assertEquals("520206", ripper.getGID(new URL("https://www.modelmayhem.com/portfolio/520206/viewall"))); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java index 72524b06..46256168 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.MyhentaicomicsRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class MyhentaicomicsRipperTest extends RippersTest { @@ -17,24 +18,24 @@ public class MyhentaicomicsRipperTest extends RippersTest { URL url = new URL("http://myhentaicomics.com/index.php/Nienna-Lost-Tales"); MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(url); // Test a comic - assertEquals("Nienna-Lost-Tales", ripper.getGID(url)); + Assertions.assertEquals("Nienna-Lost-Tales", ripper.getGID(url)); // Test a search - assertEquals("test", ripper.getGID(new URL("http://myhentaicomics.com/index.php/search?q=test"))); + Assertions.assertEquals("test", ripper.getGID(new URL("http://myhentaicomics.com/index.php/search?q=test"))); // Test a tag - assertEquals("2409", ripper.getGID(new URL("http://myhentaicomics.com/index.php/tag/2409/"))); + Assertions.assertEquals("2409", ripper.getGID(new URL("http://myhentaicomics.com/index.php/tag/2409/"))); } @Test public void testGetAlbumsToQueue() throws IOException { URL url = new URL("https://myhentaicomics.com/index.php/tag/3167/"); MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(url); - assertEquals(15, ripper.getAlbumsToQueue(ripper.getFirstPage()).size()); + Assertions.assertEquals(15, ripper.getAlbumsToQueue(ripper.getFirstPage()).size()); } @Test public void testPageContainsAlbums() throws IOException { URL url = new URL("https://myhentaicomics.com/index.php/tag/3167/"); URL url2 = new URL("https://myhentaicomics.com/index.php/search?q=test"); MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(url); - assertTrue(ripper.pageContainsAlbums(url)); - assertTrue(ripper.pageContainsAlbums(url2)); + Assertions.assertTrue(ripper.pageContainsAlbums(url)); + Assertions.assertTrue(ripper.pageContainsAlbums(url2)); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaigalleryRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaigalleryRipperTest.java index 3d126b82..19f29945 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaigalleryRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaigalleryRipperTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.MyhentaigalleryRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class MyhentaigalleryRipperTest extends RippersTest { @@ -14,9 +15,10 @@ public class MyhentaigalleryRipperTest extends RippersTest { testRipper(ripper); } + @Test public void testGetGID() throws IOException { URL url = new URL("https://myhentaigallery.com/gallery/thumbnails/9201"); MyhentaigalleryRipper ripper = new MyhentaigalleryRipper(url); - assertEquals("9201", ripper.getGID(url)); + Assertions.assertEquals("9201", ripper.getGID(url)); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NewgroundsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NewgroundsRipperTest.java index 5815aa8f..4421b267 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NewgroundsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NewgroundsRipperTest.java @@ -1,6 +1,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.NewgroundsRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -13,10 +14,11 @@ public class NewgroundsRipperTest extends RippersTest { testRipper(ripper); } + @Test public void testGetGID() throws IOException { URL url = new URL("https://zone-sama.newgrounds.com/art"); NewgroundsRipper ripper = new NewgroundsRipper(url); - assertEquals("zone-sama", ripper.getGID(url)); + Assertions.assertEquals("zone-sama", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NfsfwRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NfsfwRipperTest.java index f5bed040..00bba3b7 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NfsfwRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NfsfwRipperTest.java @@ -5,6 +5,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.NfsfwRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -21,12 +22,12 @@ public class NfsfwRipperTest extends RippersTest { public void testGetGID() throws IOException { URL url = new URL("http://nfsfw.com/gallery/v/Kitten/"); NfsfwRipper ripper = new NfsfwRipper(url); - assertEquals("Kitten", ripper.getGID(url)); + Assertions.assertEquals("Kitten", ripper.getGID(url)); url = new URL("http://nfsfw.com/gallery/v/Kitten"); - assertEquals("Kitten", ripper.getGID(url)); + Assertions.assertEquals("Kitten", ripper.getGID(url)); url = new URL("http://nfsfw.com/gallery/v/Kitten/gif_001/"); - assertEquals("Kitten__gif_001", ripper.getGID(url)); + Assertions.assertEquals("Kitten__gif_001", ripper.getGID(url)); url = new URL("http://nfsfw.com/gallery/v/Kitten/gif_001/"); - assertEquals("Kitten__gif_001", ripper.getGID(url)); + Assertions.assertEquals("Kitten__gif_001", ripper.getGID(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java index 7e82adaf..b7e1a968 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java @@ -6,6 +6,7 @@ import java.util.List; import com.rarchives.ripme.ripper.rippers.NhentaiRipper; import com.rarchives.ripme.utils.RipUtils; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class NhentaiRipperTest extends RippersTest { @@ -16,7 +17,7 @@ public class NhentaiRipperTest extends RippersTest { public void testGetGID() throws IOException { NhentaiRipper ripper = new NhentaiRipper(new URL("https://nhentai.net/g/233295/")); - assertEquals("233295", ripper.getGID(new URL("https://nhentai.net/g/233295/"))); + Assertions.assertEquals("233295", ripper.getGID(new URL("https://nhentai.net/g/233295/"))); } // Test the tag black listing @@ -28,11 +29,11 @@ public class NhentaiRipperTest extends RippersTest { // Test multiple blacklisted tags String[] tags = {"test", "one", "blowjob"}; String blacklistedTag = RipUtils.checkTags(tags, tagsOnPage); - assertEquals("blowjob", blacklistedTag); + Assertions.assertEquals("blowjob", blacklistedTag); // test tags with spaces in them String[] tags2 = {"test", "one", "sole-female"}; blacklistedTag = RipUtils.checkTags(tags2, tagsOnPage); - assertEquals("sole-female", blacklistedTag); + Assertions.assertEquals("sole-female", blacklistedTag); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java index 3353eeb5..38e697c2 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java @@ -4,15 +4,19 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.NudeGalsRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; public class NudeGalsRipperTest extends RippersTest { + @Test public void testRip() throws IOException { NudeGalsRipper ripper = new NudeGalsRipper(new URL("https://nude-gals.com/photoshoot.php?photoshoot_id=5541")); testRipper(ripper); } + @Test public void testGetGID() throws IOException { NudeGalsRipper ripper = new NudeGalsRipper(new URL("https://nude-gals.com/photoshoot.php?photoshoot_id=5541")); - assertEquals("5541", ripper.getGID( new URL("https://nude-gals.com/photoshoot.php?photoshoot_id=5541"))); + Assertions.assertEquals("5541", ripper.getGID( new URL("https://nude-gals.com/photoshoot.php?photoshoot_id=5541"))); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java index 84094515..d0e8dd6a 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java @@ -7,6 +7,7 @@ import com.rarchives.ripme.ripper.rippers.PornhubRipper; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; import org.jsoup.nodes.Document; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class PornhubRipperTest extends RippersTest { @@ -21,27 +22,26 @@ public class PornhubRipperTest extends RippersTest { public void testGetGID() throws IOException { URL url = new URL("https://www.pornhub.com/album/15680522?page=2"); PornhubRipper ripper = new PornhubRipper(url); - assertEquals("15680522", ripper.getGID(url)); + Assertions.assertEquals("15680522", ripper.getGID(url)); url = new URL("https://www.pornhub.com/album/15680522"); - assertEquals("15680522", ripper.getGID(url)); + Assertions.assertEquals("15680522", ripper.getGID(url)); } - // alternate album, with only 2 pages: https://www.pornhub.com/album/4771891 @Test public void testGetNextPage() throws IOException { - String baseURL = "https://www.pornhub.com/album/15680522"; + String baseURL = "https://www.pornhub.com/album/43902391"; PornhubRipper ripper = new PornhubRipper(new URL(baseURL)); Document page = Http.url(baseURL).get(); - int numPagesRemaining = 4; + int numPagesRemaining = 1; for (int idx = 0; idx < numPagesRemaining; idx++){ page = ripper.getNextPage(page); - assertEquals(baseURL + "?page=" + (idx + 2), page.location()); + Assertions.assertEquals(baseURL + "?page=" + (idx + 2), page.location()); } try { page = ripper.getNextPage(page); - fail("Get next page did not throw an exception on the last page"); + Assertions.fail("Get next page did not throw an exception on the last page"); } catch(IOException e){ - assertEquals(e.getMessage(), "No more pages"); + Assertions.assertEquals(e.getMessage(), "No more pages"); } } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java index 6677c02c..321afa09 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java @@ -131,11 +131,6 @@ public class RippersTest { Assertions.assertEquals(expected, actual); } - @Deprecated - void assertEquals(Object expected, Object actual) { - Assertions.assertEquals(expected, actual); - } - @Deprecated void fail(String message) { Assertions.fail(message); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java index 129fedd5..89bd0fac 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.Rule34Ripper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class Rule34RipperTest extends RippersTest { @@ -13,10 +14,11 @@ public class Rule34RipperTest extends RippersTest { testRipper(ripper); } + @Test public void testGetGID() throws IOException { URL url = new URL("https://rule34.xxx/index.php?page=post&s=list&tags=bimbo"); Rule34Ripper ripper = new Rule34Ripper(url); - assertEquals("bimbo", ripper.getGID(url)); + Assertions.assertEquals("bimbo", ripper.getGID(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RulePornRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RulePornRipperTest.java index fb23b19a..03bbaa53 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RulePornRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RulePornRipperTest.java @@ -4,16 +4,20 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.RulePornRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; public class RulePornRipperTest extends RippersTest { + @Test public void testRip() throws IOException { - RulePornRipper ripper = new RulePornRipper(new URL("https://ruleporn.com/are-you-going-to-fill-my-lil-pussy-up/")); + RulePornRipper ripper = new RulePornRipper(new URL("https://ruleporn.com/tosh/")); testRipper(ripper); } + @Test public void testGetGID() throws IOException { - URL url = new URL("https://ruleporn.com/are-you-going-to-fill-my-lil-pussy-up/"); + URL url = new URL("https://ruleporn.com/tosh/"); RulePornRipper ripper = new RulePornRipper(url); - assertEquals("are-you-going-to-fill-my-lil-pussy-up", ripper.getGID(url)); + Assertions.assertEquals("tosh", ripper.getGID(url)); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java index 035d7767..88f59e2e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java @@ -5,6 +5,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.SankakuComplexRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -29,13 +30,13 @@ public class SankakuComplexRipperTest extends RippersTest { public void testgetGID() throws IOException { URL url = new URL("https://idol.sankakucomplex.com/?tags=meme_%28me%21me%21me%21%29_%28cosplay%29"); SankakuComplexRipper ripper = new SankakuComplexRipper(url); - assertEquals("idol._meme_(me!me!me!)_(cosplay)", ripper.getGID(url)); + Assertions.assertEquals("idol._meme_(me!me!me!)_(cosplay)", ripper.getGID(url)); } @Test public void testgetSubDomain() throws IOException { URL url = new URL("https://idol.sankakucomplex.com/?tags=meme_%28me%21me%21me%21%29_%28cosplay%29"); SankakuComplexRipper ripper = new SankakuComplexRipper(url); - assertEquals("idol.", ripper.getSubDomain(url)); + Assertions.assertEquals("idol.", ripper.getSubDomain(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java index 7a7ea7f0..eb3769a1 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java @@ -5,6 +5,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.ShesFreakyRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -21,6 +22,6 @@ public class ShesFreakyRipperTest extends RippersTest { public void testGetGID() throws IOException { URL url = new URL("http://www.shesfreaky.com/gallery/nicee-snow-bunny-579NbPjUcYa.html"); ShesFreakyRipper ripper = new ShesFreakyRipper(url); - assertEquals("nicee-snow-bunny-579NbPjUcYa", ripper.getGID(url)); + Assertions.assertEquals("nicee-snow-bunny-579NbPjUcYa", ripper.getGID(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java index c4f56432..2dd311f3 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java @@ -4,16 +4,20 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.SinfestRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; public class SinfestRipperTest extends RippersTest { + @Test public void testRip() throws IOException { SinfestRipper ripper = new SinfestRipper(new URL("http://sinfest.net/view.php?date=2000-01-17")); testRipper(ripper); } + @Test public void testGetGID() throws IOException { URL url = new URL("http://sinfest.net/view.php?date=2000-01-17"); SinfestRipper ripper = new SinfestRipper(url); - assertEquals("2000-01-17", ripper.getGID(url)); + Assertions.assertEquals("2000-01-17", ripper.getGID(url)); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java index c7aa694e..cb1a78c4 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java @@ -4,16 +4,20 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.SmuttyRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; public class SmuttyRipperTest extends RippersTest { + @Test public void testRip() throws IOException { SmuttyRipper ripper = new SmuttyRipper(new URL("https://smutty.com/user/QUIGON/")); testRipper(ripper); } + @Test public void testGetGID() throws IOException { URL url = new URL("https://smutty.com/user/QUIGON/"); SmuttyRipper ripper = new SmuttyRipper(url); - assertEquals("QUIGON", ripper.getGID(url)); + Assertions.assertEquals("QUIGON", ripper.getGID(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java index 4884b205..0ba05343 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java @@ -5,22 +5,23 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.StaRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class StaRipperTest extends RippersTest { @Test - @Disabled("404 Link") + @Disabled("Ripper broken, Nullpointer exception") public void testRip() throws IOException { - StaRipper ripper = new StaRipper(new URL("https://sta.sh/2hn9rtavr1g")); + StaRipper ripper = new StaRipper(new URL("https://sta.sh/01umpyuxi4js")); testRipper(ripper); } @Test - @Disabled("404 Link") + @Disabled public void testGetGID() throws IOException { - URL url = new URL("https://sta.sh/2hn9rtavr1g"); + URL url = new URL("https://sta.sh/01umpyuxi4js"); StaRipper ripper = new StaRipper(url); - assertEquals("2hn9rtavr1g", ripper.getGID(url)); + Assertions.assertEquals("01umpyuxi4js", ripper.getGID(url)); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TapasticRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TapasticRipperTest.java index bd424860..36a3a29e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TapasticRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TapasticRipperTest.java @@ -4,18 +4,22 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.TapasticRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class TapasticRipperTest extends RippersTest { @Test + @Disabled("ripper broken") public void testTapasticRip() throws IOException { - TapasticRipper ripper = new TapasticRipper(new URL("https://tapas.io/series/tsiwbakd-comic")); + TapasticRipper ripper = new TapasticRipper(new URL("https://tapas.io/series/TPIAG")); testRipper(ripper); } + @Test public void testGetGID() throws IOException { - URL url = new URL("https://tapas.io/series/tsiwbakd-comic"); + URL url = new URL("https://tapas.io/series/TPIAG"); TapasticRipper ripper = new TapasticRipper(url); - assertEquals("series_ tsiwbakd-comic", ripper.getGID(url)); + Assertions.assertEquals("series_ TPIAG", ripper.getGID(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java index 4d088742..6d27ca7c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.TeenplanetRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class TeenplanetRipperTest extends RippersTest { @@ -13,9 +14,10 @@ public class TeenplanetRipperTest extends RippersTest { testRipper(ripper); } + @Test public void testGetGID() throws IOException { URL url = new URL("http://teenplanet.org/galleries/the-perfect-side-of-me-6588.html"); TeenplanetRipper ripper = new TeenplanetRipper(url); - assertEquals("the-perfect-side-of-me-6588", ripper.getGID(url)); + Assertions.assertEquals("the-perfect-side-of-me-6588", ripper.getGID(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java index 65ebbb14..ce8ce88c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.TheyiffgalleryRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class TheyiffgalleryRipperTest extends RippersTest { @@ -13,9 +14,10 @@ public class TheyiffgalleryRipperTest extends RippersTest { testRipper(ripper); } + @Test public void testGetGID() throws IOException { URL url = new URL("https://theyiffgallery.com/index?/category/4303"); TheyiffgalleryRipper ripper = new TheyiffgalleryRipper(url); - assertEquals("4303", ripper.getGID(url)); + Assertions.assertEquals("4303", ripper.getGID(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java index f76e2b25..58decfe6 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.VidbleRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class VidbleRipperTest extends RippersTest { @@ -13,10 +14,11 @@ public class VidbleRipperTest extends RippersTest { testRipper(ripper); } + @Test public void testGetGID() throws IOException { URL url = new URL("http://www.vidble.com/album/y1oyh3zd"); VidbleRipper ripper = new VidbleRipper(url); - assertEquals("y1oyh3zd", ripper.getGID(url)); + Assertions.assertEquals("y1oyh3zd", ripper.getGID(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java index 327698bd..b7f52e99 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java @@ -5,6 +5,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.VkRipper; import org.json.JSONObject; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class VkRipperTest extends RippersTest { @@ -35,7 +36,7 @@ public class VkRipperTest extends RippersTest { String responseJson = "{\"id\":\"-45984105_457345201\",\"base\":\"https://sun9-37.userapi.com/\",\"tagged\":[],\"likes\":0,\"shares\":0,\"o_src\":\"https://sun9-65.userapi.com/c857520/v857520962/10e24c/DPxygc3XW5E.jpg\",\"o_\":[\"https://sun9-65.userapi.com/c857520/v857520962/10e24c/DPxygc3XW5E\",130,98],\"z_src\":\"https://sun9-41.userapi.com/c857520/v857520962/10e24a/EsDDQA36qKI.jpg\",\"z_\":[\"https://sun9-41.userapi.com/c857520/v857520962/10e24a/EsDDQA36qKI\",1280,960],\"w_src\":\"https://sun9-60.userapi.com/c857520/v857520962/10e24b/6ETsA15rAdU.jpg\",\"w_\":[\"https://sun9-60.userapi.com/c857520/v857520962/10e24b/6ETsA15rAdU\",1405,1054]}"; - assertTrue( + Assertions.assertTrue( ripper.findJSONObjectContainingPhotoId("-45984105_457345201", new JSONObject(json)) .similar(new JSONObject(responseJson))); } @@ -45,7 +46,7 @@ public class VkRipperTest extends RippersTest { VkRipper ripper = new VkRipper(new URL("http://vk.com/album45506334_0")); String json = "{\"id\":\"-45984105_457345201\",\"base\":\"https://sun9-37.userapi.com/\",\"commcount\":0,\"date\":\"3 Dec at 1:14 am\",\"tagged\":[],\"attached_tags\":{\"max_tags_per_object\":5},\"o_src\":\"https://sun9-65.userapi.com/c857520/v857520962/10e24c/DPxygc3XW5E.jpg\",\"o_\":[\"https://sun9-65.userapi.com/c857520/v857520962/10e24c/DPxygc3XW5E\",130,98],\"y_src\":\"https://sun9-9.userapi.com/c857520/v857520962/10e249/dUDeuY10s0A.jpg\",\"y_\":[\"https://sun9-9.userapi.com/c857520/v857520962/10e249/dUDeuY10s0A\",807,605],\"z_src\":\"https://sun9-41.userapi.com/c857520/v857520962/10e24a/EsDDQA36qKI.jpg\",\"z_\":[\"https://sun9-41.userapi.com/c857520/v857520962/10e24a/EsDDQA36qKI\",1280,960]}"; - assertEquals("https://sun9-41.userapi.com/c857520/v857520962/10e24a/EsDDQA36qKI.jpg", + Assertions.assertEquals("https://sun9-41.userapi.com/c857520/v857520962/10e24a/EsDDQA36qKI.jpg", ripper.getBestSourceUrl(new JSONObject(json))); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java index ad634d5f..6f4ed2eb 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java @@ -4,23 +4,26 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.WebtoonsRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class WebtoonsRipperTest extends RippersTest { @Test public void testWebtoonsAlbum() throws IOException { - WebtoonsRipper ripper = new WebtoonsRipper(new URL("http://www.webtoons.com/en/drama/my-boo/ep-33/viewer?title_no=1185&episode_no=33")); + WebtoonsRipper ripper = new WebtoonsRipper(new URL("https://www.webtoons.com/en/super-hero/unordinary/episode-103/viewer?title_no=679&episode_no=109")); testRipper(ripper); } @Test - public void testWebtoonsType() throws IOException { + public void testWedramabtoonsType() throws IOException { WebtoonsRipper ripper = new WebtoonsRipper(new URL("http://www.webtoons.com/en/drama/lookism/ep-145/viewer?title_no=1049&episode_no=145")); testRipper(ripper); } @Test + @Disabled("URL format different") public void testGetGID() throws IOException { - URL url = new URL("http://www.webtoons.com/en/drama/my-boo/ep-33/viewer?title_no=1185&episode_no=33"); + URL url = new URL("https://www.webtoons.com/en/super-hero/unordinary/episode-103/viewer?title_no=679&episode_no=109"); WebtoonsRipper ripper = new WebtoonsRipper(url); - assertEquals("my-boo", ripper.getGID(url)); + Assertions.assertEquals("super-hero", ripper.getGID(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java index f55e5fdc..cad697de 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java @@ -5,6 +5,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.WordpressComicRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -66,7 +67,7 @@ public class WordpressComicRipperTest extends RippersTest { public void test_konradokonski_getAlbumTitle() throws IOException { URL url = new URL("http://www.konradokonski.com/sawdust/comic/get-up/"); WordpressComicRipper ripper = new WordpressComicRipper(url); - assertEquals("konradokonski.com_sawdust", ripper.getAlbumTitle(url)); + Assertions.assertEquals("konradokonski.com_sawdust", ripper.getAlbumTitle(url)); } @@ -93,7 +94,7 @@ public class WordpressComicRipperTest extends RippersTest { public void test_Eightmuses_getAlbumTitle() throws IOException { URL url = new URL("https://8muses.download/lustomic-playkittens-josh-samuel-porn-comics-8-muses/"); WordpressComicRipper ripper = new WordpressComicRipper(url); - assertEquals("8muses.download_lustomic-playkittens-josh-samuel-porn-comics-8-muses", ripper.getAlbumTitle(url)); + Assertions.assertEquals("8muses.download_lustomic-playkittens-josh-samuel-porn-comics-8-muses", ripper.getAlbumTitle(url)); } @Test public void test_spyingwithlana_download() throws IOException { @@ -105,7 +106,7 @@ public class WordpressComicRipperTest extends RippersTest { public void test_spyingwithlana_getAlbumTitle() throws IOException { URL url = new URL("http://spyingwithlana.com/comic/the-big-hookup/"); WordpressComicRipper ripper = new WordpressComicRipper(url); - assertEquals("spyingwithlana_the-big-hookup", ripper.getAlbumTitle(url)); + Assertions.assertEquals("spyingwithlana_the-big-hookup", ripper.getAlbumTitle(url)); } @Test diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java index 96a9295e..433ad0d0 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java @@ -5,6 +5,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.XhamsterRipper; import org.jsoup.nodes.Document; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -48,7 +49,7 @@ public class XhamsterRipperTest extends RippersTest { public void testGetGID() throws IOException { URL url = new URL("https://xhamster.com/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664"); XhamsterRipper ripper = new XhamsterRipper(url); - assertEquals("7254664", ripper.getGID(url)); + Assertions.assertEquals("7254664", ripper.getGID(url)); } @Test public void testGetNextPage() throws IOException { diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java index 7cc3def9..ec95a02c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.YuvutuRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class YuvutuRipperTest extends RippersTest { @@ -21,6 +22,6 @@ public class YuvutuRipperTest extends RippersTest { public void testGetGID() throws IOException { URL url = new URL("http://www.yuvutu.com/modules.php?name=YuGallery&action=view&set_id=420333"); YuvutuRipper ripper = new YuvutuRipper(url); - assertEquals("420333", ripper.getGID(url)); + Assertions.assertEquals("420333", ripper.getGID(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java index e12ccb5c..7af10e55 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ZizkiRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class ZizkiRipperTest extends RippersTest { @@ -15,12 +16,12 @@ public class ZizkiRipperTest extends RippersTest { public void testGetGID() throws IOException { URL url = new URL("http://zizki.com/dee-chorde/we-got-spirit"); ZizkiRipper ripper = new ZizkiRipper(url); - assertEquals("dee-chorde", ripper.getGID(url)); + Assertions.assertEquals("dee-chorde", ripper.getGID(url)); } @Test public void testAlbumTitle() throws IOException { URL url = new URL("http://zizki.com/dee-chorde/we-got-spirit"); ZizkiRipper ripper = new ZizkiRipper(url); - assertEquals("zizki_Dee Chorde_We Got Spirit", ripper.getAlbumTitle(url)); + Assertions.assertEquals("zizki_Dee Chorde_We Got Spirit", ripper.getAlbumTitle(url)); } } From e57b5db64ffd5178030172411543c43978a4af00 Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 29 Jul 2020 07:36:10 +0200 Subject: [PATCH 048/512] chainrippertest method does nothing, remove --- .../ripme/tst/ripper/rippers/ChanRipperTest.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java index 57c56fe3..e7b285fc 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java @@ -14,19 +14,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class ChanRipperTest extends RippersTest { - @Test - public void testChanURLFailures() throws IOException { - List failURLs = new ArrayList<>(); - // URLs that should not work - for (URL url : failURLs) { - try { - new ChanRipper(url); - Assertions.fail("Instantiated ripper for URL that should not work: " + url); - } catch (Exception e) { - // Expected - } - } - } @Test public void testChanURLPasses() throws IOException { List passURLs = new ArrayList<>(); From 9ef257ce60fc5070e72a6b5eece5cf6595226fda Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 29 Jul 2020 08:04:38 +0200 Subject: [PATCH 049/512] remove deprecated assertEquals(expected,actual) --- .../ripme/tst/ripper/rippers/ErofusRipperTest.java | 6 +++++- .../ripme/tst/ripper/rippers/EromeRipperTest.java | 5 +++-- .../ripme/tst/ripper/rippers/FolioRipperTest.java | 3 ++- .../tst/ripper/rippers/FuraffinityRipperTest.java | 6 ++++-- .../ripper/rippers/GfycatporntubeRipperTest.java | 6 +++++- .../tst/ripper/rippers/HentaifoundryRipperTest.java | 3 ++- .../tst/ripper/rippers/HqpornerRipperTest.java | 5 +++-- .../tst/ripper/rippers/HypnohubRipperTest.java | 10 ++++++++-- .../tst/ripper/rippers/ImagefapRipperTest.java | 3 ++- .../tst/ripper/rippers/ImagevenueRipperTest.java | 3 ++- .../ripme/tst/ripper/rippers/ImgboxRipperTest.java | 4 +++- .../ripme/tst/ripper/rippers/ImgurRipperTest.java | 13 +++++++------ .../tst/ripper/rippers/InstagramRipperTest.java | 5 ++++- .../tst/ripper/rippers/KingcomixRipperTest.java | 5 ++++- .../tst/ripper/rippers/PhotobucketRipperTest.java | 13 +++++++------ .../tst/ripper/rippers/PicstatioRipperTest.java | 3 ++- .../ripme/tst/ripper/rippers/RippersTest.java | 5 ----- 17 files changed, 63 insertions(+), 35 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java index ab497433..6acd6e08 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java @@ -4,15 +4,19 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ErofusRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; public class ErofusRipperTest extends RippersTest { + @Test public void testRip() throws IOException { ErofusRipper ripper = new ErofusRipper(new URL("https://www.erofus.com/comics/be-story-club-comics/a-kiss/issue-1")); testRipper(ripper); } + @Test public void testGetGID() throws IOException { ErofusRipper ripper = new ErofusRipper(new URL("https://www.erofus.com/comics/be-story-club-comics/a-kiss/issue-1")); - assertEquals("be-story-club-comics", ripper.getGID(new URL("https://www.erofus.com/comics/be-story-club-comics/a-kiss/issue-1"))); + Assertions.assertEquals("be-story-club-comics", ripper.getGID(new URL("https://www.erofus.com/comics/be-story-club-comics/a-kiss/issue-1"))); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java index 4e7241af..b18762e9 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java @@ -5,6 +5,7 @@ import java.net.MalformedURLException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.EromeRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class EromeRipperTest extends RippersTest { @@ -12,13 +13,13 @@ public class EromeRipperTest extends RippersTest { public void testGetGIDProfilePage() throws IOException { URL url = new URL("https://www.erome.com/Jay-Jenna"); EromeRipper ripper = new EromeRipper(url); - assertEquals("Jay-Jenna", ripper.getGID(url)); + Assertions.assertEquals("Jay-Jenna", ripper.getGID(url)); } @Test public void testGetGIDAlbum() throws IOException { URL url = new URL("https://www.erome.com/a/KbDAM1XT"); EromeRipper ripper = new EromeRipper(url); - assertEquals("KbDAM1XT", ripper.getGID(url)); + Assertions.assertEquals("KbDAM1XT", ripper.getGID(url)); } @Test public void testGetAlbumsToQueue() throws IOException { diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java index 66a9c1ed..3a8627bd 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.FolioRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class FolioRipperTest extends RippersTest { @@ -21,6 +22,6 @@ public class FolioRipperTest extends RippersTest { public void testGetGID() throws IOException { URL url = new URL("https://folio.ink/DmBe6i"); FolioRipper ripper = new FolioRipper(url); - assertEquals("DmBe6i", ripper.getGID(url)); + Assertions.assertEquals("DmBe6i", ripper.getGID(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java index 1f397bcb..1efc30f7 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.FuraffinityRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class FuraffinityRipperTest extends RippersTest { @@ -18,17 +19,18 @@ public class FuraffinityRipperTest extends RippersTest { testRipper(ripper); } + @Test public void testGetGID() throws IOException { URL url = new URL("https://www.furaffinity.net/gallery/mustardgas/"); FuraffinityRipper ripper = new FuraffinityRipper(url); - assertEquals("mustardgas", ripper.getGID(url)); + Assertions.assertEquals("mustardgas", ripper.getGID(url)); } @Test public void testLogin() throws IOException { URL url = new URL("https://www.furaffinity.net/gallery/mustardgas/"); FuraffinityRipper ripper = new FuraffinityRipper(url); // Check if the first page contain the username of ripmes shared account - Boolean containsUsername = ripper.getFirstPage().html().contains("ripmethrowaway"); + boolean containsUsername = ripper.getFirstPage().html().contains("ripmethrowaway"); assert containsUsername; } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java index e53c78e6..18068b18 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java @@ -4,16 +4,20 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.GfycatporntubeRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; public class GfycatporntubeRipperTest extends RippersTest { + @Test public void testRip() throws IOException { GfycatporntubeRipper ripper = new GfycatporntubeRipper(new URL("https://gfycatporntube.com/blowjob-bunny-puts-on-a-show/")); testRipper(ripper); } + @Test public void testGetGID() throws IOException { URL url = new URL("https://gfycatporntube.com/blowjob-bunny-puts-on-a-show/"); GfycatporntubeRipper ripper = new GfycatporntubeRipper(url); - assertEquals("blowjob-bunny-puts-on-a-show", ripper.getGID(url)); + Assertions.assertEquals("blowjob-bunny-puts-on-a-show", ripper.getGID(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java index d86e2904..7623c61d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.HentaifoundryRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class HentaifoundryRipperTest extends RippersTest { @@ -16,7 +17,7 @@ public class HentaifoundryRipperTest extends RippersTest { public void testHentaifoundryGetGID() throws IOException { HentaifoundryRipper ripper = new HentaifoundryRipper(new URL("https://www.hentai-foundry.com/stories/user/Rakked")); testRipper(ripper); - assertEquals("Rakked", ripper.getGID(new URL("https://www.hentai-foundry.com/stories/user/Rakked"))); + Assertions.assertEquals("Rakked", ripper.getGID(new URL("https://www.hentai-foundry.com/stories/user/Rakked"))); } @Test public void testHentaifoundryPdfRip() throws IOException { diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HqpornerRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HqpornerRipperTest.java index 6ef21c8d..c6aebc83 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HqpornerRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HqpornerRipperTest.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.HqpornerRipper; import com.rarchives.ripme.utils.Utils; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -20,7 +21,7 @@ public class HqpornerRipperTest extends RippersTest { public void testGetGID() throws IOException { URL poolURL = new URL("https://hqporner.com/hdporn/84636-pool_lesson_with_a_cheating_husband.html"); HqpornerRipper ripper = new HqpornerRipper(poolURL); - assertEquals("84636-pool_lesson_with_a_cheating_husband", ripper.getGID(poolURL)); + Assertions.assertEquals("84636-pool_lesson_with_a_cheating_husband", ripper.getGID(poolURL)); } @Test public void testGetURLsFromPage() throws IOException { @@ -39,7 +40,7 @@ public class HqpornerRipperTest extends RippersTest { try { ripper.getNextPage(ripper.getFirstPage()); } catch (IOException e) { - assertEquals(e.getMessage(), "No next page found."); + Assertions.assertEquals(e.getMessage(), "No next page found."); } } @Test diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java index 1d9ef4ad..001b3d63 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java @@ -4,8 +4,13 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.HypnohubRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; public class HypnohubRipperTest extends RippersTest { + @Test + @Disabled("wants a hunman") public void testRip() throws IOException { URL poolURL = new URL("http://hypnohub.net/pool/show/2303"); URL postURL = new URL("http://hypnohub.net/post/show/63464/black_hair-bracelet-collar-corruption-female_only-"); @@ -14,12 +19,13 @@ public class HypnohubRipperTest extends RippersTest { ripper = new HypnohubRipper(postURL); testRipper(ripper); } + @Test public void testGetGID() throws IOException { URL poolURL = new URL("http://hypnohub.net/pool/show/2303"); HypnohubRipper ripper = new HypnohubRipper(poolURL); - assertEquals("2303", ripper.getGID(poolURL)); + Assertions.assertEquals("2303", ripper.getGID(poolURL)); URL postURL = new URL("http://hypnohub.net/post/show/63464/black_hair-bracelet-collar-corruption-female_only-"); - assertEquals("63464_black_hair-bracelet-collar-corruption-female_only-", ripper.getGID(postURL)); + Assertions.assertEquals("63464_black_hair-bracelet-collar-corruption-female_only-", ripper.getGID(postURL)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java index 533fff79..6a112a5d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java @@ -6,6 +6,7 @@ import java.util.HashMap; import java.util.Map; import com.rarchives.ripme.ripper.rippers.ImagefapRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class ImagefapRipperTest extends RippersTest { @@ -30,6 +31,6 @@ public class ImagefapRipperTest extends RippersTest { public void testImagefapGetAlbumTitle() throws IOException { URL url = new URL("https://www.imagefap.com/gallery.php?gid=7789753"); ImagefapRipper ripper = new ImagefapRipper(url); - assertEquals("imagefap_Red.Heels.Lover.In.Love_7789753", ripper.getAlbumTitle(url)); + Assertions.assertEquals("imagefap_Red.Heels.Lover.In.Love_7789753", ripper.getAlbumTitle(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagevenueRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagevenueRipperTest.java index 8093ede7..43d211a7 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagevenueRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagevenueRipperTest.java @@ -5,6 +5,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.ImagevenueRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -21,6 +22,6 @@ public class ImagevenueRipperTest extends RippersTest { public void testGetGID() throws IOException { URL url = new URL("http://img120.imagevenue.com/galshow.php?gal=gallery_1373818527696_191lo"); ImagevenueRipper ripper = new ImagevenueRipper(url); - assertEquals("gallery_1373818527696_191lo", ripper.getGID(url)); + Assertions.assertEquals("gallery_1373818527696_191lo", ripper.getGID(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java index bb877450..3b6bb782 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ImgboxRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class ImgboxRipperTest extends RippersTest { @@ -13,9 +14,10 @@ public class ImgboxRipperTest extends RippersTest { testRipper(ripper); } + @Test public void testGetGID() throws IOException { URL url = new URL("https://imgbox.com/g/FJPF7t26FD"); ImgboxRipper ripper = new ImgboxRipper(url); - assertEquals("FJPF7t26FD", ripper.getGID(url)); + Assertions.assertEquals("FJPF7t26FD", ripper.getGID(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java index 5c4fb1a1..3772d5f2 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java @@ -4,6 +4,7 @@ import com.rarchives.ripme.ripper.rippers.ImgurRipper; import com.rarchives.ripme.ripper.rippers.ImgurRipper.ImgurAlbum; import com.rarchives.ripme.utils.RipUtils; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -25,7 +26,7 @@ public class ImgurRipperTest extends RippersTest { for (URL url : failURLs) { try { new ImgurRipper(url); - fail("Instantiated ripper for URL that should not work: " + url); + Assertions.fail("Instantiated ripper for URL that should not work: " + url); } catch (Exception e) { // Expected } @@ -69,21 +70,21 @@ public class ImgurRipperTest extends RippersTest { @Test public void testImgurAlbumWithMoreThan20Pictures() throws IOException { ImgurAlbum album = ImgurRipper.getImgurAlbum(new URL("http://imgur.com/a/HUMsq")); - assertTrue("Failed to find 20 files from " + album.url.toExternalForm() + ", only got " + album.images.size(), - album.images.size() >= 20); + Assertions.assertTrue(album.images.size() >= 20, + "Failed to find 20 files from " + album.url.toExternalForm() + ", only got " + album.images.size()); } @Test public void testImgurAlbumWithMoreThan100Pictures() throws IOException { ImgurAlbum album = ImgurRipper.getImgurAlbum(new URL("https://imgur.com/a/HX3JSrD")); - assertTrue("Failed to find 100 files from " + album.url.toExternalForm() + ", only got " + album.images.size(), - album.images.size() >= 100); + Assertions.assertTrue(album.images.size() >= 100, + "Failed to find 100 files from " + album.url.toExternalForm() + ", only got " + album.images.size()); } @Test public void testImgurVideoFromGetFilesFromURL() throws Exception { List urls = RipUtils.getFilesFromURL(new URL("https://i.imgur.com/4TtwxRN.gifv")); - assertEquals("https://i.imgur.com/4TtwxRN.mp4", urls.get(0).toExternalForm()); + Assertions.assertEquals("https://i.imgur.com/4TtwxRN.mp4", urls.get(0).toExternalForm()); } /* diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java index 693dc72f..85b3b248 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.InstagramRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -28,12 +30,13 @@ public class InstagramRipperTest extends RippersTest { for (URL url : testURLs.keySet()) { InstagramRipper ripper = new InstagramRipper(url); ripper.setup(); - assertEquals(testURLs.get(url), ripper.getGID(ripper.getURL())); + Assertions.assertEquals(testURLs.get(url), ripper.getGID(ripper.getURL())); deleteDir(ripper.getWorkingDir()); } } @Test + @Disabled("Ripper broken for single items") public void testInstagramAlbums() throws IOException { List contentURLs = new ArrayList<>(); // This unit test is a bit flaky diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/KingcomixRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/KingcomixRipperTest.java index ebe23e4c..bea92e8b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/KingcomixRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/KingcomixRipperTest.java @@ -5,11 +5,14 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.KingcomixRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class KingcomixRipperTest extends RippersTest { @Test + @Disabled("test or ripper broken") public void testRip() throws IOException { KingcomixRipper ripper = new KingcomixRipper(new URL("https://kingcomix.com/aunt-cumming-tracy-scops/")); testRipper(ripper); @@ -19,7 +22,7 @@ public class KingcomixRipperTest extends RippersTest { public void testGetGID() throws IOException { URL url = new URL("https://kingcomix.com/aunt-cumming-tracy-scops/"); KingcomixRipper ripper = new KingcomixRipper(url); - assertEquals("aunt-cumming-tracy-scops", ripper.getGID(url)); + Assertions.assertEquals("aunt-cumming-tracy-scops", ripper.getGID(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java index b2d409d3..fb133d32 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java @@ -5,6 +5,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.PhotobucketRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -39,9 +40,9 @@ public class PhotobucketRipperTest extends RippersTest { } try { page = ripper.getNextPage(page); - fail("Get next page did not throw an exception on the last page"); + Assertions.fail("Get next page did not throw an exception on the last page"); } catch (IOException e) { - assertEquals(e.getMessage(), "No more pages"); + Assertions.assertEquals(e.getMessage(), "No more pages"); } } @@ -50,13 +51,13 @@ public class PhotobucketRipperTest extends RippersTest { URL url = new URL( "http://s732.photobucket.com/user/doublesix66/library/Army%20Painter%20examples?sort=3&page=1"); PhotobucketRipper ripper = new PhotobucketRipper(url); - assertEquals("doublesix66", ripper.getGID(url)); + Assertions.assertEquals("doublesix66", ripper.getGID(url)); url = new URL( "http://s732.photobucket.com/user/doublesix66/library/Army%20Painter%20examples/Painting%20examples?page=1&sort=3"); - assertEquals("doublesix66", ripper.getGID(url)); + Assertions.assertEquals("doublesix66", ripper.getGID(url)); url = new URL("http://s844.photobucket.com/user/SpazzySpizzy/library/Album%20Covers"); - assertEquals("SpazzySpizzy", ripper.getGID(url)); + Assertions.assertEquals("SpazzySpizzy", ripper.getGID(url)); url = new URL("http://s844.photobucket.com/user/SpazzySpizzy/library"); - assertEquals("SpazzySpizzy", ripper.getGID(url)); + Assertions.assertEquals("SpazzySpizzy", ripper.getGID(url)); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PicstatioRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PicstatioRipperTest.java index 1c5cf273..ec4e2383 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PicstatioRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PicstatioRipperTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.PicstatioRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class PicstatioRipperTest extends RippersTest { @@ -15,6 +16,6 @@ public class PicstatioRipperTest extends RippersTest { @Test public void testGID() throws IOException { PicstatioRipper ripper = new PicstatioRipper(new URL("https://www.picstatio.com/aerial-view-wallpapers")); - assertEquals("aerial-view-wallpapers", ripper.getGID(new URL("https://www.picstatio.com/aerial-view-wallpapers"))); + Assertions.assertEquals("aerial-view-wallpapers", ripper.getGID(new URL("https://www.picstatio.com/aerial-view-wallpapers"))); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java index 321afa09..be3bf8f0 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java @@ -126,11 +126,6 @@ public class RippersTest { } } - @Deprecated - void assertEquals(String expected, String actual) { - Assertions.assertEquals(expected, actual); - } - @Deprecated void fail(String message) { Assertions.fail(message); From b9ef190c3d405921b23ecef2bdcc1b7b3a985b4f Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 29 Jul 2020 08:17:14 +0200 Subject: [PATCH 050/512] fix tests --- .../com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java | 3 +++ .../ripme/tst/ripper/rippers/ComicextraRipperTest.java | 2 ++ .../rarchives/ripme/tst/ripper/rippers/DribbbleRipperTest.java | 2 ++ .../rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java | 3 +++ .../rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java | 2 ++ 5 files changed, 12 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java index 2ade8754..3ceb2ac7 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java @@ -6,11 +6,13 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.BatoRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class BatoRipperTest extends RippersTest { @Test + @Disabled("cloudlare? gets unavailable in test but works in browser") public void testRip() throws IOException { BatoRipper ripper = new BatoRipper(new URL("https://bato.to/chapter/1207152")); testRipper(ripper); @@ -24,6 +26,7 @@ public class BatoRipperTest extends RippersTest { } @Test + @Disabled("cloudlare? gets unavailable in test but works in browser") public void testGetAlbumTitle() throws IOException { URL url = new URL("https://bato.to/chapter/1207152"); BatoRipper ripper = new BatoRipper(url); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java index ffe0f12b..0769e295 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java @@ -3,6 +3,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ComicextraRipper; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class ComicextraRipperTest extends RippersTest { @@ -13,6 +14,7 @@ public class ComicextraRipperTest extends RippersTest { testRipper(ripper); } @Test + @Disabled("no images found error, broken ripper?") public void testChapterUrl() throws IOException { URL url = new URL("https://www.comicextra.com/v-for-vendetta/chapter-1"); ComicextraRipper ripper = new ComicextraRipper(url); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DribbbleRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DribbbleRipperTest.java index 36b56d6a..bd4321fc 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DribbbleRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DribbbleRipperTest.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.DribbbleRipper; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class DribbbleRipperTest extends RippersTest { @Test + @Disabled("test or ripper broken") public void testDribbbleRip() throws IOException { DribbbleRipper ripper = new DribbbleRipper(new URL("https://dribbble.com/typogriff")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java index 334cd6ff..e73f35b4 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java @@ -4,15 +4,18 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.FuskatorRipper; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class FuskatorRipperTest extends RippersTest { @Test + @Disabled("test or ripper broken") public void testFuskatorAlbum() throws IOException { FuskatorRipper ripper = new FuskatorRipper(new URL("https://fuskator.com/thumbs/hqt6pPXAf9z/Shaved-Blonde-Babe-Katerina-Ambre.html")); testRipper(ripper); } @Test + @Disabled("test or ripper broken") public void testUrlsWithTiled() throws IOException { FuskatorRipper ripper = new FuskatorRipper(new URL("https://fuskator.com/thumbs/hsrzk~UIFmJ/Blonde-Babe-Destiny-Dixon-Playing-With-Black-Dildo.html")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java index 433ad0d0..27b8364a 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java @@ -6,6 +6,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.XhamsterRipper; import org.jsoup.nodes.Document; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -36,6 +37,7 @@ public class XhamsterRipperTest extends RippersTest { testRipper(ripper); } @Test + @Disabled("ripper broken?") public void testXhamsterVideo() throws IOException { XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/videos/brazzers-busty-big-booty-milf-lisa-ann-fucks-her-masseur-1492828")); testRipper(ripper); From e2e6aca9658d917c18959c96688c0592ce897aeb Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 29 Jul 2020 08:18:02 +0200 Subject: [PATCH 051/512] mulemax renamed to fooktube --- .../{MulemaxRipper.java => FooktubeRipper.java} | 12 ++++++------ .../tst/ripper/rippers/FooktubeRipperTest.java | 16 ++++++++++++++++ .../tst/ripper/rippers/MulemaxRipperTest.java | 16 ---------------- 3 files changed, 22 insertions(+), 22 deletions(-) rename src/main/java/com/rarchives/ripme/ripper/rippers/{MulemaxRipper.java => FooktubeRipper.java} (82%) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/FooktubeRipperTest.java delete mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/MulemaxRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MulemaxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FooktubeRipper.java similarity index 82% rename from src/main/java/com/rarchives/ripme/ripper/rippers/MulemaxRipper.java rename to src/main/java/com/rarchives/ripme/ripper/rippers/FooktubeRipper.java index 01bf4b1c..3cda70b2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MulemaxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FooktubeRipper.java @@ -16,11 +16,11 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.VideoRipper; import com.rarchives.ripme.utils.Http; -public class MulemaxRipper extends AbstractSingleFileRipper { +public class FooktubeRipper extends AbstractSingleFileRipper { private static final String HOST = "mulemax"; - public MulemaxRipper(URL url) throws IOException { + public FooktubeRipper(URL url) throws IOException { super(url); } @@ -41,7 +41,7 @@ public class MulemaxRipper extends AbstractSingleFileRipper { @Override public boolean canRip(URL url) { - Pattern p = Pattern.compile("^https?://.*mulemax\\.com/video/(.*)/.*$"); + Pattern p = Pattern.compile("^https?://.*fooktube\\.com/video/(.*)/.*$"); Matcher m = p.matcher(url.toExternalForm()); return m.matches(); } @@ -53,15 +53,15 @@ public class MulemaxRipper extends AbstractSingleFileRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://.*mulemax\\.com/video/(.*)/(.*)$"); + Pattern p = Pattern.compile("^https?://.*fooktube\\.com/video/(.*)/(.*)$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(2); } throw new MalformedURLException( - "Expected mulemax format:" - + "mulemax.com/video/####" + "Expected fooktube format:" + + "fooktube.com/video/####" + " Got: " + url); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FooktubeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FooktubeRipperTest.java new file mode 100644 index 00000000..b98cabdd --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FooktubeRipperTest.java @@ -0,0 +1,16 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.FooktubeRipper; +import org.junit.jupiter.api.Test; + +public class FooktubeRipperTest extends RippersTest { + @Test + public void testFooktubeVideo() throws IOException { + FooktubeRipper ripper = new FooktubeRipper(new URL("https://fooktube.com/video/641/in-the-cinema")); //pick any video from the front page + testRipper(ripper); + } + +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MulemaxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MulemaxRipperTest.java deleted file mode 100644 index 8c12a94d..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MulemaxRipperTest.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URL; - -import com.rarchives.ripme.ripper.rippers.MulemaxRipper; -import org.junit.jupiter.api.Test; - -public class MulemaxRipperTest extends RippersTest { - @Test - public void testMulemaxVideo() throws IOException { - MulemaxRipper ripper = new MulemaxRipper(new URL("https://mulemax.com/video/1720/emma-and-her-older-sissy-are-home-for-a-holiday-break")); //pick any video from the front page - testRipper(ripper); - } - -} \ No newline at end of file From 6a47a393fc9920d0cde96886280906bfd1aa2fb0 Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 29 Jul 2020 09:10:22 +0200 Subject: [PATCH 052/512] JagodibujaRipperTest fails on ubuntu, github pr check only --- .../ripme/tst/ripper/rippers/JagodibujaRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/JagodibujaRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/JagodibujaRipperTest.java index d41c0352..af4314c8 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/JagodibujaRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/JagodibujaRipperTest.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.JagodibujaRipper; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class JagodibujaRipperTest extends RippersTest { @Test + @Disabled("fails on github ubuntu automated PR check 2020-07-29") public void testJagodibujaRipper() throws IOException { // a photo set JagodibujaRipper ripper = new JagodibujaRipper(new URL("http://www.jagodibuja.com/comic-in-me/")); From 652383640a5c07d22b23aca24156d629514bdb34 Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 29 Jul 2020 09:21:51 +0200 Subject: [PATCH 053/512] remove deprecated assertTrue --- .../tst/ripper/rippers/HitomiRipperTest.java | 3 +- .../tst/ripper/rippers/RedgifsRipperTest.java | 4 +-- .../ripme/tst/ripper/rippers/RippersTest.java | 31 +++++-------------- .../tst/ripper/rippers/VideoRippersTest.java | 5 +-- 4 files changed, 15 insertions(+), 28 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java index 00c937b3..646c7f46 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java @@ -5,6 +5,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.HitomiRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -14,6 +15,6 @@ public class HitomiRipperTest extends RippersTest { public void testRip() throws IOException { HitomiRipper ripper = new HitomiRipper(new URL("https://hitomi.la/galleries/975973.html")); testRipper(ripper); - assertTrue(ripper.getGID(new URL("https://hitomi.la/galleries/975973.html")).equals("975973")); + Assertions.assertTrue(ripper.getGID(new URL("https://hitomi.la/galleries/975973.html")).equals("975973")); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java index 922dbaf8..bcca515c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java @@ -49,8 +49,8 @@ public class RedgifsRipperTest extends RippersTest { Document doc = ripper.getFirstPage(); doc = ripper.getNextPage(doc); - assertTrue("https://napi.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=150".equalsIgnoreCase(doc.location())); + Assertions.assertTrue("https://napi.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=150".equalsIgnoreCase(doc.location())); doc = ripper.getNextPage(doc); - assertTrue("https://napi.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=300".equalsIgnoreCase(doc.location())); + Assertions.assertTrue("https://napi.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=300".equalsIgnoreCase(doc.location())); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java index be3bf8f0..fa824fe9 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java @@ -22,7 +22,7 @@ public class RippersTest { private final Logger logger = Logger.getLogger(RippersTest.class); public void testStub() { - assertTrue("RippersTest must contain at lease one test.", true); + Assertions.assertTrue(true, "RippersTest must contain at lease one test."); } void testRipper(AbstractRipper ripper) { @@ -36,18 +36,18 @@ public class RippersTest { ripper.setup(); ripper.markAsTest(); ripper.rip(); - assertTrue("Failed to download a single file from " + ripper.getURL(), - ripper.getWorkingDir().listFiles().length >= 1); + Assertions.assertTrue(ripper.getWorkingDir().listFiles().length >= 1, + "Failed to download a single file from " + ripper.getURL()); } catch (IOException e) { if (e.getMessage().contains("Ripping interrupted")) { // We expect some rips to get interrupted } else { e.printStackTrace(); - fail("Failed to rip " + ripper.getURL() + " : " + e.getMessage()); + Assertions.fail("Failed to rip " + ripper.getURL() + " : " + e.getMessage()); } } catch (Exception e) { e.printStackTrace(); - fail("Failed to rip " + ripper.getURL() + " : " + e.getMessage()); + Assertions.fail("Failed to rip " + ripper.getURL() + " : " + e.getMessage()); } finally { deleteDir(ripper.getWorkingDir()); } @@ -69,17 +69,17 @@ public class RippersTest { ripper.setup(); ripper.markAsTest(); List foundUrls = ripper.getURLsFromPage(ripper.getFirstPage()); - assertTrue("Failed to find single url on page " + ripper.getURL(), foundUrls.size() >= 1); + Assertions.assertTrue(foundUrls.size() >= 1, "Failed to find single url on page " + ripper.getURL()); } catch (IOException e) { if (e.getMessage().contains("Ripping interrupted")) { // We expect some rips to get interrupted } else { e.printStackTrace(); - fail("Failed to rip " + ripper.getURL() + " : " + e.getMessage()); + Assertions.fail("Failed to rip " + ripper.getURL() + " : " + e.getMessage()); } } catch (Exception e) { e.printStackTrace(); - fail("Failed to rip " + ripper.getURL() + " : " + e.getMessage()); + Assertions.fail("Failed to rip " + ripper.getURL() + " : " + e.getMessage()); } finally { deleteDir(ripper.getWorkingDir()); } @@ -126,21 +126,6 @@ public class RippersTest { } } - @Deprecated - void fail(String message) { - Assertions.fail(message); - } - - @Deprecated - void assertTrue(boolean condition) { - Assertions.assertTrue(condition); - } - - @Deprecated - void assertTrue(String failMessage, boolean condition) { - Assertions.assertTrue(condition, failMessage); - } - @Deprecated void assertFalse(String message, boolean condition) { Assertions.assertFalse(condition, message); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java index e8b5ffe3..fb162946 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java @@ -9,6 +9,7 @@ import com.rarchives.ripme.ripper.VideoRipper; import com.rarchives.ripme.ripper.rippers.video.PornhubRipper; import com.rarchives.ripme.ripper.rippers.video.YuvutuRipper; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -28,9 +29,9 @@ public class VideoRippersTest extends RippersTest { // Video ripper testing is... weird. // If the ripper finds the URL to download the video, and it's a test, // then the ripper sets the download URL as the ripper's URL. - assertFalse("Failed to find download url for " + oldURL, oldURL.equals(ripper.getURL())); + Assertions.assertFalse(oldURL.equals(ripper.getURL()), "Failed to find download url for " + oldURL); } catch (Exception e) { - fail("Error while ripping " + ripper.getURL() + " : " + e); + Assertions.fail("Error while ripping " + ripper.getURL() + " : " + e); e.printStackTrace(); } finally { deleteDir(ripper.getWorkingDir()); From 8e80943926adb3ddcef21ed49b44b940841c3c1e Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 29 Jul 2020 09:22:42 +0200 Subject: [PATCH 054/512] remove deprecated assertFalse, assertNull, assertNotNull --- .../ripme/tst/ripper/rippers/RippersTest.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java index fa824fe9..f1a95622 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java @@ -126,19 +126,4 @@ public class RippersTest { } } - @Deprecated - void assertFalse(String message, boolean condition) { - Assertions.assertFalse(condition, message); - } - - @Deprecated - void assertNull(Object actual) { - Assertions.assertNull(actual); - } - - @Deprecated - void assertNotNull(String message, Object actual) { - Assertions.assertNotNull(actual, message); - } - } From 0bce4be2272b258142826ffc831468226651738c Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 29 Jul 2020 09:41:05 +0200 Subject: [PATCH 055/512] make it work with java-14, upgrade to jacoco-0.8.5 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 977176f2..92da5597 100644 --- a/pom.xml +++ b/pom.xml @@ -126,7 +126,7 @@ org.jacoco jacoco-maven-plugin - 0.8.2 + 0.8.5 prepare-agent @@ -165,4 +165,4 @@ - \ No newline at end of file + From f639758a74bd1f06efa5d0eab2ae619df8d745bc Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 29 Jul 2020 17:55:51 -0400 Subject: [PATCH 056/512] 1.7.93: Fixed Motherless ripper; Fixed e621 ripper; Updated pt_PT translation; Implemented redgifs Ripper; added missing translation to Korean/KR; Fixed elecx ripper; Added ripper for HentaiNexus --- pom.xml | 2 +- ripme.json | 5 +++-- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 92da5597..ac9399ff 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.rarchives.ripme ripme jar - 1.7.92 + 1.7.93 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index e80f6b55..1c7cd314 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "currentHash": "e0d946933c0a89da03e9a5a00420a87dd8c2fc23eae01e1a2fb8390eceb9b868", + "currentHash": "357639535c112ca857117151321141cb08ad313021079d522c2c5d15ee799998", "changeList": [ + "1.7.93: Fixed Motherless ripper; Fixed e621 ripper; Updated pt_PT translation; Implemented redgifs Ripper; added missing translation to Korean/KR; Fixed elecx ripper; Added ripper for HentaiNexus", "1.7.92: Added read-comic.com ripper; Fix Pawoo ripper; Add ChineseSimplified language file; Fixed artstation ripper", "1.7.91: Fixed luscious ripper. Fixed VK ripper; Added Kingcomix ripper", "1.7.90: Added FitnakedgirlsRipper; Fixed VK Album Ripper; Fixed Myreadingmanga Ripper; Fixed windows max file name; Fixed Pornhub Video Ripper; Fixed Motherless Ripper; Fixed Instagram Ripper", @@ -264,5 +265,5 @@ "1.0.3: Added VK.com ripper", "1.0.1: Added auto-update functionality" ], - "latestVersion": "1.7.92" + "latestVersion": "1.7.93" } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index a09730ce..88dc6a17 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -23,7 +23,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.92"; + private static final String DEFAULT_VERSION = "1.7.93"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static String mainFileName; From eae9ecf37b8725bd2362c659d4581e0277d059b0 Mon Sep 17 00:00:00 2001 From: Sheefip <61481253+Sheefip@users.noreply.github.com> Date: Sun, 2 Aug 2020 14:22:20 +0200 Subject: [PATCH 057/512] RedditRipper: added gallery support A gallery is a special kind of reddit post, which can have up to 20 images. They can be recognized by the is_gallery field in the JSON. The gallery_data array describes the structure of the gallery by referencing the image ids, it also contains the captions and links, which are ignored by this implementation. The s field of each entry in media_metadata array appears to reference the highest quality image. getJsonURL and getGID were modified to handle reddit.com/gallery/id links. parseJsonChild was modified to handle the gallery case. The new method handleGallery parses gallery_data and media_metadata and adds the images to download. A test for a gallery was added. --- .../ripme/ripper/rippers/RedditRipper.java | 44 ++++++++++++++++++- .../tst/ripper/rippers/RedditRipperTest.java | 7 +++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index e68e477d..3d2fe22c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -10,6 +10,7 @@ import java.util.regex.Pattern; import com.rarchives.ripme.ui.RipStatusMessage; import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import org.json.JSONTokener; @@ -54,6 +55,13 @@ public class RedditRipper extends AlbumRipper { } private URL getJsonURL(URL url) throws MalformedURLException { + // Convert gallery to post link and append ".json" + Pattern p = Pattern.compile("^https?://[a-zA-Z0-9.]{0,4}reddit\\.com/gallery/([a-zA-Z0-9]+).*$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return new URL("https://reddit.com/" +m.group(m.groupCount())+ ".json"); + } + // Append ".json" to URL in appropriate location. String result = url.getProtocol() + "://" + url.getHost() + url.getPath() + ".json"; if (url.getQuery() != null) { @@ -188,6 +196,8 @@ public class RedditRipper extends AlbumRipper { if (data.getBoolean("is_self")) { // TODO Parse self text handleBody(data.getString("selftext"), data.getString("id"), data.getString("title")); + } else if (data.has("is_gallery") && data.getBoolean("is_gallery")) { + handleGallery(data.getJSONObject("gallery_data").getJSONArray("items"), data.getJSONObject("media_metadata"), data.getString("id"), data.getString("title")); } else { // Get link handleURL(data.getString("url"), data.getString("id"), data.getString("title")); @@ -291,6 +301,31 @@ public class RedditRipper extends AlbumRipper { } } + private void handleGallery(JSONArray data, JSONObject metadata, String id, String title){ + //TODO handle captions and caption urls + String subdirectory = ""; + if (Utils.getConfigBoolean("reddit.use_sub_dirs", true)) { + if (Utils.getConfigBoolean("album_titles.save", true)) { + subdirectory = title; + title = "-" + title + "-"; + } + } + for (int i = 0; i < data.length(); i++) { + JSONObject media = metadata.getJSONObject(data.getJSONObject(i).getString("media_id")); + String prefix = id + "-"; + if (Utils.getConfigBoolean("download.save_order", true)) { + //announcement says up to 20 (https://www.reddit.com/r/announcements/comments/hrrh23/now_you_can_make_posts_with_multiple_images/) + prefix += String.format("%02d-", i + 1); + } + try { + URL mediaURL = new URL(media.getJSONObject("s").getString("u").replaceAll("&", "&")); + addURLToDownload(mediaURL, prefix, subdirectory); + } catch (MalformedURLException | JSONException e) { + LOGGER.error("[!] Unable to parse gallery JSON:\ngallery_data:\n" + data +"\nmedia_metadata:\n" + metadata); + } + } + } + @Override public String getHost() { return HOST; @@ -312,6 +347,13 @@ public class RedditRipper extends AlbumRipper { return "post_" + m.group(m.groupCount()); } + // Gallery + p = Pattern.compile("^https?://[a-zA-Z0-9.]{0,4}reddit\\.com/gallery/([a-zA-Z0-9]+).*$"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return "post_" + m.group(m.groupCount()); + } + // Subreddit p = Pattern.compile("^https?://[a-zA-Z0-9.]{0,4}reddit\\.com/r/([a-zA-Z0-9_]+).*$"); m = p.matcher(url.toExternalForm()); @@ -319,7 +361,7 @@ public class RedditRipper extends AlbumRipper { return "sub_" + m.group(m.groupCount()); } - throw new MalformedURLException("Only accepts user pages, subreddits, or post, can't understand " + url); + throw new MalformedURLException("Only accepts user pages, subreddits, post, or gallery can't understand " + url); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java index d5d9600d..972e2f9b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java @@ -56,4 +56,11 @@ public class RedditRipperTest extends RippersTest { new URL("https://www.reddit.com/r/bottesting/comments/7msmhi/bad_link/")); testRipper(ripper); } + + @Test + public void testRedditGallery() throws IOException{ + RedditRipper ripper = new RedditRipper( + new URL("https://www.reddit.com/gallery/hrrh23")); + testRipper(ripper); + } } From 029422b41725ab54f8af7d036180843784e41609 Mon Sep 17 00:00:00 2001 From: cariah Date: Wed, 5 Aug 2020 17:25:42 +0200 Subject: [PATCH 058/512] fix: instagramRipper, replaced Nashorn with GraalVM.js --- pom.xml | 5 ++ .../ripme/ripper/rippers/InstagramRipper.java | 68 +++++++++---------- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/pom.xml b/pom.xml index ac9399ff..60e08138 100644 --- a/pom.xml +++ b/pom.xml @@ -42,6 +42,11 @@ jsoup 1.8.1 + + org.graalvm.js + js + 20.1.0 + org.json json diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index f231def4..82b7dea5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -1,18 +1,13 @@ package com.rarchives.ripme.ripper.rippers; +import com.oracle.js.parser.ErrorManager; +import com.oracle.js.parser.Parser; +import com.oracle.js.parser.ScriptEnvironment; +import com.oracle.js.parser.Source; +import com.oracle.js.parser.ir.*; import com.rarchives.ripme.ripper.AbstractJSONRipper; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; -import jdk.nashorn.internal.ir.Block; -import jdk.nashorn.internal.ir.CallNode; -import jdk.nashorn.internal.ir.ExpressionStatement; -import jdk.nashorn.internal.ir.FunctionNode; -import jdk.nashorn.internal.ir.Statement; -import jdk.nashorn.internal.parser.Parser; -import jdk.nashorn.internal.runtime.Context; -import jdk.nashorn.internal.runtime.ErrorManager; -import jdk.nashorn.internal.runtime.Source; -import jdk.nashorn.internal.runtime.options.Options; import org.json.JSONArray; import org.json.JSONObject; import org.jsoup.Connection; @@ -26,12 +21,7 @@ import java.time.Instant; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Spliterators; +import java.util.*; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; @@ -176,13 +166,17 @@ public class InstagramRipper extends AbstractJSONRipper { if (postRip) { return null; } - Predicate hrefFilter = (storiesRip || pinnedReelRip) ? href -> href.contains("Consumer.js") : - href -> href.contains("ProfilePageContainer.js") || href.contains("TagPageContainer.js"); + + Predicate hrefFilter = href -> href.contains("Consumer.js"); + if (taggedRip) { + hrefFilter = href -> href.contains("ProfilePageContainer.js") || href.contains("TagPageContainer.js"); + } String href = doc.select("link[rel=preload]").stream() - .map(link -> link.attr("href")) - .filter(hrefFilter) - .findFirst().orElse(""); + .map(link -> link.attr("href")) + .filter(hrefFilter) + .findFirst().orElse(""); + String body = Http.url("https://www.instagram.com" + href).cookies(cookies).response().body(); Function hashExtractor = @@ -386,7 +380,7 @@ public class InstagramRipper extends AbstractJSONRipper { case "GraphSidecar": JSONArray sideCar = getJsonArrayByPath(mediaItem, "edge_sidecar_to_children.edges"); return getStreamOfJsonArray(sideCar).map(object -> object.getJSONObject("node")) - .flatMap(this::parseRootForUrls); + .flatMap(this::parseRootForUrls); default: return Stream.empty(); } @@ -415,18 +409,19 @@ public class InstagramRipper extends AbstractJSONRipper { /* ------------------------------------------------------------------------------------------------------- */ private String getHashValue(String javaScriptData, String keyword, int offset) { List statements = getJsBodyBlock(javaScriptData).getStatements(); + return statements.stream() - .flatMap(statement -> filterItems(statement, ExpressionStatement.class)) - .map(ExpressionStatement::getExpression) - .flatMap(expression -> filterItems(expression, CallNode.class)) - .map(CallNode::getArgs) - .map(expressions -> expressions.get(0)) - .flatMap(expression -> filterItems(expression, FunctionNode.class)) - .map(FunctionNode::getBody) - .map(Block::getStatements) - .map(statementList -> lookForHash(statementList, keyword, offset)) - .filter(Objects::nonNull) - .findFirst().orElse(null); + .flatMap(statement -> filterItems(statement, ExpressionStatement.class)) + .map(ExpressionStatement::getExpression) + .flatMap(expression -> filterItems(expression, CallNode.class)) + .map(CallNode::getArgs) + .map(expressions -> expressions.get(0)) + .flatMap(expression -> filterItems(expression, FunctionNode.class)) + .map(FunctionNode::getBody) + .map(Block::getStatements) + .map(statementList -> lookForHash(statementList, keyword, offset)) + .filter(Objects::nonNull) + .findFirst().orElse(null); } private String lookForHash(List list, String keyword, int offset) { @@ -444,9 +439,10 @@ public class InstagramRipper extends AbstractJSONRipper { } private Block getJsBodyBlock(String javaScriptData) { - ErrorManager errors = new ErrorManager(); - Context context = new Context(new Options("nashorn"), errors, Thread.currentThread().getContextClassLoader()); - return new Parser(context.getEnv(), Source.sourceFor("name", javaScriptData), errors).parse().getBody(); + ScriptEnvironment env = ScriptEnvironment.builder().ecmaScriptVersion(10).constAsVar(true).build(); + ErrorManager errorManager = new ErrorManager.ThrowErrorManager(); + Source src = Source.sourceFor("name", javaScriptData); + return new Parser(env, src, errorManager).parse().getBody(); } // Some JSON helper methods below From 8118287cddec976c27f84a1c26470958a842e12d Mon Sep 17 00:00:00 2001 From: Sheefip <61481253+Sheefip@users.noreply.github.com> Date: Fri, 14 Aug 2020 10:19:36 +0200 Subject: [PATCH 059/512] RedditRipper: changed gallery condition --- .../java/com/rarchives/ripme/ripper/rippers/RedditRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 3d2fe22c..65d854fb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -196,7 +196,7 @@ public class RedditRipper extends AlbumRipper { if (data.getBoolean("is_self")) { // TODO Parse self text handleBody(data.getString("selftext"), data.getString("id"), data.getString("title")); - } else if (data.has("is_gallery") && data.getBoolean("is_gallery")) { + } else if (!data.isNull("gallery_data") && !data.isNull("media_metadata")) { handleGallery(data.getJSONObject("gallery_data").getJSONArray("items"), data.getJSONObject("media_metadata"), data.getString("id"), data.getString("title")); } else { // Get link From 5785b91de356adff2f20a10413c5726e579902af Mon Sep 17 00:00:00 2001 From: jeff-up Date: Thu, 10 Sep 2020 19:47:15 -0500 Subject: [PATCH 060/512] Pawoo no longer has a custom theme --- .../rarchives/ripme/ripper/rippers/PawooRipper.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PawooRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PawooRipper.java index 31817c84..100068ed 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PawooRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PawooRipper.java @@ -23,16 +23,4 @@ public class PawooRipper extends MastodonRipper { return "pawoo.net"; } - - @Override - // Pawoo uses a custom theme that has different navigation links - public Document getNextPage(Document doc) throws IOException { - Elements hrefs = doc.select(".pagination a[rel=\"next\"]"); - if (hrefs.isEmpty()) { - throw new IOException("No more pages"); - } - String nextUrl = hrefs.last().attr("href"); - sleep(500); - return Http.url(nextUrl).get(); - } } From ee4db54c3e2e0536d927f71ebc6ec8b89519572f Mon Sep 17 00:00:00 2001 From: pwnstr <71965491+pwnstr@users.noreply.github.com> Date: Sun, 27 Sep 2020 10:26:36 +0100 Subject: [PATCH 061/512] Fix early termination bug in cookiesForURL The condition for the while loop was causing the config cookie search to break after only checking the full domain, instead of trying higher-level domains. For example www.imagefap.com would be tried but not imagefap.com. --- src/main/java/com/rarchives/ripme/utils/Http.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index 1b85005c..fb90bbd8 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -81,8 +81,9 @@ public class Http { while (parts.length > 1) { String domain = String.join(".", parts); // Try to get cookies for this host from config + logger.info("Trying to load cookies from config for " + domain); cookieStr = Utils.getConfigString("cookies." + domain, ""); - if (cookieStr.equals("")) { + if (!cookieStr.equals("")) { cookieDomain = domain; // we found something, start parsing break; From 2bb8d3a64fbb5987375843d7d41e038a4aaae2af Mon Sep 17 00:00:00 2001 From: nikozark <37566071+nikozark@users.noreply.github.com> Date: Sat, 3 Oct 2020 05:23:02 +0300 Subject: [PATCH 062/512] Update MangadexRipper.java Added functionality to rip entire manga --- .../ripme/ripper/rippers/MangadexRipper.java | 92 ++++++++++++++++--- 1 file changed, 79 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java index 6697a45b..ea8c4530 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java @@ -1,24 +1,28 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractJSONRipper; +import com.rarchives.ripme.ui.History; +import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; +import org.apache.log4j.Logger; import org.json.JSONArray; import org.json.JSONObject; import org.jsoup.Connection; import org.jsoup.nodes.Document; +import java.io.File; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import java.util.ArrayList; -import java.util.List; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; public class MangadexRipper extends AbstractJSONRipper { private String chapterApiEndPoint = "https://mangadex.org/api/chapter/"; - + private String mangaApiEndPoint = "https://mangadex.org/api/manga/"; + private boolean isSingleChapter; private String getImageUrl(String chapterHash, String imageName, String server) { return server + chapterHash + "/" + imageName; } @@ -44,41 +48,102 @@ public class MangadexRipper extends AbstractJSONRipper { @Override public String getGID(URL url) throws MalformedURLException { String capID = getChapterID(url.toExternalForm()); + String mangaID = getMangaID(url.toExternalForm()); if (capID != null) { + isSingleChapter=true; return capID; } + else + if(mangaID!=null){ + isSingleChapter=false; + return mangaID; + } throw new MalformedURLException("Unable to get chapter ID from" + url); } private String getChapterID(String url) { - Pattern p = Pattern.compile("https://mangadex.org/chapter/([\\d]+)/?"); + Pattern p = Pattern.compile("https://mangadex.org/chapter/([\\d]+)/([\\d+]?)"); Matcher m = p.matcher(url); if (m.matches()) { return m.group(1); } return null; } + private String getMangaID(String url){ + Pattern p = Pattern.compile("https://mangadex.org/title/([\\d]+)/(.+)"); + Matcher m = p.matcher(url); + if(m.matches()){ + return m.group(1); + } + return null; + } + @Override public JSONObject getFirstPage() throws IOException { // Get the chapter ID String chapterID = getChapterID(url.toExternalForm()); - return Http.url(new URL(chapterApiEndPoint + chapterID)).getJSON(); + String mangaID = getMangaID(url.toExternalForm()); + if(mangaID!=null){ + return Http.url(new URL(mangaApiEndPoint+mangaID)).getJSON(); + } + else + return Http.url(new URL(chapterApiEndPoint + chapterID)).getJSON(); } @Override protected List getURLsFromJSON(JSONObject json) { + if(isSingleChapter){ + List assetURLs = new ArrayList<>(); + JSONArray currentObject; + String chapterHash; + // Server is the cdn hosting the images. + String server; + chapterHash = json.getString("hash"); + server = json.getString("server"); + for (int i = 0; i < json.getJSONArray("page_array").length(); i++) { + currentObject = json.getJSONArray("page_array"); + + assetURLs.add(getImageUrl(chapterHash, currentObject.getString(i), server)); + } + return assetURLs; + } + JSONObject chaptersJSON = (JSONObject) json.get("chapter"); + JSONObject temp; + Iterator keys = chaptersJSON.keys(); + HashMap chapterIDs = new HashMap<>(); + while (keys.hasNext()) { + String keyValue = (String) keys.next(); + temp=(JSONObject)chaptersJSON.get(keyValue); + if(temp.getString("lang_name").equals("English")) { + chapterIDs.put(temp.getDouble("chapter"),keyValue); + } + + } + List assetURLs = new ArrayList<>(); JSONArray currentObject; - - String chapterHash = json.getString("hash"); + String chapterHash; // Server is the cdn hosting the images. - String server = json.getString("server"); + String server; + JSONObject chapterJSON=null; + TreeMap treeMap = new TreeMap<>(chapterIDs); + Iterator it = treeMap.keySet().iterator(); + while(it.hasNext()) { + double key =(double) it.next(); + try { + chapterJSON = Http.url(new URL(chapterApiEndPoint + treeMap.get(key))).getJSON(); + } catch (IOException e) { + e.printStackTrace(); + } + sendUpdate(RipStatusMessage.STATUS.LOADING_RESOURCE,"chapter "+key); + chapterHash = chapterJSON.getString("hash"); + server = chapterJSON.getString("server"); + for (int i = 0; i < chapterJSON.getJSONArray("page_array").length(); i++) { + currentObject = chapterJSON.getJSONArray("page_array"); - for (int i = 0; i < json.getJSONArray("page_array").length(); i++) { - currentObject = json.getJSONArray("page_array"); - - assetURLs.add(getImageUrl(chapterHash, currentObject.getString(i), server)); + assetURLs.add(getImageUrl(chapterHash, currentObject.getString(i), server)); + } } return assetURLs; @@ -91,4 +156,5 @@ public class MangadexRipper extends AbstractJSONRipper { addURLToDownload(url, getPrefix(index)); } -} + +} \ No newline at end of file From 18d8b51a948b1533b411ed8b3b9e9dabcfd6618c Mon Sep 17 00:00:00 2001 From: nikozark <37566071+nikozark@users.noreply.github.com> Date: Sat, 3 Oct 2020 05:36:15 +0300 Subject: [PATCH 063/512] Update MangadexRipperTest.java Added test case for manga --- .../ripme/tst/ripper/rippers/MangadexRipperTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MangadexRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MangadexRipperTest.java index 70aa5aaf..62aabf7c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MangadexRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MangadexRipperTest.java @@ -11,5 +11,12 @@ public class MangadexRipperTest extends RippersTest{ MangadexRipper ripper = new MangadexRipper(new URL("https://mangadex.org/chapter/467904/")); testRipper(ripper); } + public class testMangaRip extends RippersTest{ + + public void testRip() throws IOException { + MangadexRipper ripper = new MangadexRipper(new URL("https://mangadex.org/title/44625/this-croc-will-die-in-100-days")); + testRipper(ripper); + } + } } From 15919e906fca60959e5ba9fad218ac66d5dfdc62 Mon Sep 17 00:00:00 2001 From: agg23 Date: Mon, 5 Oct 2020 09:11:59 -0700 Subject: [PATCH 064/512] Added support for ripping entire Flickr profiles --- .../ripme/ripper/rippers/FlickrRipper.java | 106 +++++++++++++----- 1 file changed, 76 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java index e56cb4a1..32088424 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java @@ -10,6 +10,7 @@ import java.util.regex.Pattern; import com.rarchives.ripme.ui.RipStatusMessage; import org.json.JSONArray; import org.json.JSONObject; +import org.json.JSONException; import org.jsoup.nodes.Document; import com.rarchives.ripme.ripper.AbstractHTMLRipper; @@ -21,6 +22,22 @@ public class FlickrRipper extends AbstractHTMLRipper { private Document albumDoc = null; private final DownloadThreadPool flickrThreadPool; + + private enum UrlType { + USER, + PHOTOSET + } + + private class Album { + final UrlType type; + final String id; + + Album(UrlType type, String id) { + this.type = type; + this.id = id; + } + } + @Override public DownloadThreadPool getThreadPool() { return flickrThreadPool; @@ -81,40 +98,44 @@ public class FlickrRipper extends AbstractHTMLRipper { } // The flickr api is a monster of weird settings so we just request everything that the webview does - private String apiURLBuilder(String photoset, String pageNumber, String apiKey) { - LOGGER.info("https://api.flickr.com/services/rest?extras=can_addmeta," + - "can_comment,can_download,can_share,contact,count_comments,count_faves,count_views,date_taken," + - "date_upload,icon_urls_deep,isfavorite,ispro,license,media,needs_interstitial,owner_name," + - "owner_datecreate,path_alias,realname,rotation,safety_level,secret_k,secret_h,url_c,url_f,url_h,url_k," + - "url_l,url_m,url_n,url_o,url_q,url_s,url_sq,url_t,url_z,visibility,visibility_source,o_dims," + - "is_marketplace_printable,is_marketplace_licensable,publiceditability&per_page=100&page="+ pageNumber + "&" + - "get_user_info=1&primary_photo_extras=url_c,%20url_h,%20url_k,%20url_l,%20url_m,%20url_n,%20url_o" + - ",%20url_q,%20url_s,%20url_sq,%20url_t,%20url_z,%20needs_interstitial,%20can_share&jump_to=&" + - "photoset_id=" + photoset + "&viewerNSID=&method=flickr.photosets.getPhotos&csrf=&" + - "api_key=" + apiKey + "&format=json&hermes=1&hermesClient=1&reqId=358ed6a0&nojsoncallback=1"); + private String apiURLBuilder(Album album, String pageNumber, String apiKey) { + String method = null; + String idField = null; + switch (album.type) { + case PHOTOSET: + method = "flickr.photosets.getPhotos"; + idField = "photoset_id=" + album.id; + break; + case USER: + method = "flickr.people.getPhotos"; + idField = "user_id=" + album.id; + break; + } + return "https://api.flickr.com/services/rest?extras=can_addmeta," + - "can_comment,can_download,can_share,contact,count_comments,count_faves,count_views,date_taken," + - "date_upload,icon_urls_deep,isfavorite,ispro,license,media,needs_interstitial,owner_name," + - "owner_datecreate,path_alias,realname,rotation,safety_level,secret_k,secret_h,url_c,url_f,url_h,url_k," + - "url_l,url_m,url_n,url_o,url_q,url_s,url_sq,url_t,url_z,visibility,visibility_source,o_dims," + - "is_marketplace_printable,is_marketplace_licensable,publiceditability&per_page=100&page="+ pageNumber + "&" + - "get_user_info=1&primary_photo_extras=url_c,%20url_h,%20url_k,%20url_l,%20url_m,%20url_n,%20url_o" + - ",%20url_q,%20url_s,%20url_sq,%20url_t,%20url_z,%20needs_interstitial,%20can_share&jump_to=&" + - "photoset_id=" + photoset + "&viewerNSID=&method=flickr.photosets.getPhotos&csrf=&" + - "api_key=" + apiKey + "&format=json&hermes=1&hermesClient=1&reqId=358ed6a0&nojsoncallback=1"; + "can_comment,can_download,can_share,contact,count_comments,count_faves,count_views,date_taken," + + "date_upload,icon_urls_deep,isfavorite,ispro,license,media,needs_interstitial,owner_name," + + "owner_datecreate,path_alias,realname,rotation,safety_level,secret_k,secret_h,url_c,url_f,url_h,url_k," + + "url_l,url_m,url_n,url_o,url_q,url_s,url_sq,url_t,url_z,visibility,visibility_source,o_dims," + + "is_marketplace_printable,is_marketplace_licensable,publiceditability&per_page=100&page="+ pageNumber + "&" + + "get_user_info=1&primary_photo_extras=url_c,%20url_h,%20url_k,%20url_l,%20url_m,%20url_n,%20url_o" + + ",%20url_q,%20url_s,%20url_sq,%20url_t,%20url_z,%20needs_interstitial,%20can_share&jump_to=&" + + idField + "&viewerNSID=&method=" + method + "&csrf=&" + + "api_key=" + apiKey + "&format=json&hermes=1&hermesClient=1&reqId=358ed6a0&nojsoncallback=1"; } private JSONObject getJSON(String page, String apiKey) { URL pageURL = null; String apiURL = null; try { - apiURL = apiURLBuilder(getPhotosetID(url.toExternalForm()), page, apiKey); + apiURL = apiURLBuilder(getAlbum(url.toExternalForm()), page, apiKey); pageURL = new URL(apiURL); } catch (MalformedURLException e) { LOGGER.error("Unable to get api link " + apiURL + " is malformed"); } try { - LOGGER.info(Http.url(pageURL).ignoreContentType().get().text()); + LOGGER.info("Fetching: " + apiURL); + LOGGER.info("Response: " + Http.url(pageURL).ignoreContentType().get().text()); return new JSONObject(Http.url(pageURL).ignoreContentType().get().text()); } catch (IOException e) { LOGGER.error("Unable to get api link " + apiURL + " is malformed"); @@ -122,21 +143,32 @@ public class FlickrRipper extends AbstractHTMLRipper { } } - private String getPhotosetID(String url) { + private Album getAlbum(String url) throws MalformedURLException { Pattern p; Matcher m; - // Root: https://www.flickr.com/photos/115858035@N04/ + // User photostream: https://www.flickr.com/photos/115858035@N04/ // Album: https://www.flickr.com/photos/115858035@N04/sets/72157644042355643/ final String domainRegex = "https?://[wm.]*flickr.com"; final String userRegex = "[a-zA-Z0-9@_-]+"; // Album - p = Pattern.compile("^" + domainRegex + "/photos/(" + userRegex + ")/(sets|albums)/([0-9]+)/?.*$"); + p = Pattern.compile("^" + domainRegex + "/photos/" + userRegex + "/(sets|albums)/([0-9]+)/?.*$"); m = p.matcher(url); if (m.matches()) { - return m.group(3); + return new Album(UrlType.PHOTOSET, m.group(2)); } - return null; + + // User photostream + p = Pattern.compile("^" + domainRegex + "/photos/(" + userRegex + ")/?$"); + m = p.matcher(url); + if (m.matches()) { + return new Album(UrlType.USER, m.group(1)); + } + + String errorMessage = "Failed to extract photoset ID from url: " + url; + + LOGGER.error(errorMessage); + throw new MalformedURLException(errorMessage); } @Override @@ -144,7 +176,7 @@ public class FlickrRipper extends AbstractHTMLRipper { if (!url.toExternalForm().contains("/sets/")) { return super.getAlbumTitle(url); } - try { + try { // Attempt to use album title as GID Document doc = getFirstPage(); String user = url.toExternalForm(); @@ -214,9 +246,23 @@ public class FlickrRipper extends AbstractHTMLRipper { if (jsonData.has("stat") && jsonData.getString("stat").equals("fail")) { break; } else { - int totalPages = jsonData.getJSONObject("photoset").getInt("pages"); + // Determine root key + JSONObject rootData; + + try { + rootData = jsonData.getJSONObject("photoset"); + } catch (JSONException e) { + try { + rootData = jsonData.getJSONObject("photos"); + } catch (JSONException innerE) { + LOGGER.error("Unable to find photos in response"); + break; + } + } + + int totalPages = rootData.getInt("pages"); LOGGER.info(jsonData); - JSONArray pictures = jsonData.getJSONObject("photoset").getJSONArray("photo"); + JSONArray pictures = rootData.getJSONArray("photo"); for (int i = 0; i < pictures.length(); i++) { LOGGER.info(i); JSONObject data = (JSONObject) pictures.get(i); From 096ad4a655c5a88d00597fe9b73dff1c415f76de Mon Sep 17 00:00:00 2001 From: Tushar Date: Fri, 9 Oct 2020 04:52:17 +0530 Subject: [PATCH 065/512] Fix ig ripper grabbing incorrect query hash --- .../ripme/ripper/rippers/InstagramRipper.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index 82b7dea5..4a4122ad 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -192,7 +192,8 @@ public class InstagramRipper extends AbstractJSONRipper { } private String getProfileHash(String jsData) { - return getHashValue(jsData, "loadProfilePageExtras", -1); + return getHashValue(jsData, "loadProfilePageExtras", -1, + s -> s.replaceAll(".*queryId\\s?:\\s?\"([0-9a-f]*)\".*", "$1")); } private String getPinnedHash(String jsData) { @@ -407,7 +408,8 @@ public class InstagramRipper extends AbstractJSONRipper { // Javascript parsing /* ------------------------------------------------------------------------------------------------------- */ - private String getHashValue(String javaScriptData, String keyword, int offset) { + private String getHashValue(String javaScriptData, String keyword, int offset, + Function extractHash) { List statements = getJsBodyBlock(javaScriptData).getStatements(); return statements.stream() @@ -419,15 +421,22 @@ public class InstagramRipper extends AbstractJSONRipper { .flatMap(expression -> filterItems(expression, FunctionNode.class)) .map(FunctionNode::getBody) .map(Block::getStatements) - .map(statementList -> lookForHash(statementList, keyword, offset)) + .map(statementList -> lookForHash(statementList, keyword, offset, extractHash)) .filter(Objects::nonNull) .findFirst().orElse(null); } - private String lookForHash(List list, String keyword, int offset) { + private String getHashValue(String javaScriptData, String keyword, int offset) { + return getHashValue(javaScriptData, keyword, offset, null); + } + + private String lookForHash(List list, String keyword, int offset, Function extractHash) { for (int i = 0; i < list.size(); i++) { Statement st = list.get(i); if (st.toString().contains(keyword)) { + if (extractHash != null) { + return extractHash.apply(list.get(i + offset).toString()); + } return list.get(i + offset).toString().replaceAll(".*\"([0-9a-f]*)\".*", "$1"); } } From 922acd8c47df1ed0b49fee06633306d6ac3e1df5 Mon Sep 17 00:00:00 2001 From: Anon Date: Tue, 13 Oct 2020 05:59:48 +0200 Subject: [PATCH 066/512] macOS installation docs added --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index dd6438f7..63299485 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,13 @@ Download `ripme.jar` from the [latest release](https://github.com/ripmeapp/ripme For information about running the `.jar` file, see [the How To Run wiki](https://github.com/ripmeapp/ripme/wiki/How-To-Run-RipMe). +## Installation + +On macOS, there is a [cask](https://github.com/Homebrew/homebrew-cask/blob/master/Casks/ripme.rb). +``` +breww cask install ripme && xattr -d com.apple.quarantine /Applications/ripme.jar +``` + ## Changelog [Changelog](https://github.com/ripmeapp/ripme/blob/master/ripme.json) **(ripme.json)** From 34316362e48a542e39c2e098f06845524e54dd7f Mon Sep 17 00:00:00 2001 From: 12-Seconds <59331191+12-Seconds@users.noreply.github.com> Date: Sun, 25 Oct 2020 11:48:43 +0400 Subject: [PATCH 067/512] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 63299485..1e88b5d0 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ For information about running the `.jar` file, see [the How To Run wiki](https:/ On macOS, there is a [cask](https://github.com/Homebrew/homebrew-cask/blob/master/Casks/ripme.rb). ``` -breww cask install ripme && xattr -d com.apple.quarantine /Applications/ripme.jar +brew cask install ripme && xattr -d com.apple.quarantine /Applications/ripme.jar ``` ## Changelog From 118c353f2dd3af8a6d4a5681a4707e1124f3675a Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 30 Jul 2020 02:55:01 +0200 Subject: [PATCH 068/512] tag slow tests, so they can be excluded/included as the runtime of the tests can be long, tag the slow ones so they can be excluded more easy on the command line or configuration. see here for hints: https://mkyong.com/junit5/junit-5-tagging-and-filtering-tag-examples/ https://stackoverflow.com/questions/60474190/how-to-include-exclude-junit5-tags-in-gradle-cmd to run tests with tag slow only, or not run tags with tag slow do, quoting the expression might be necessary depending on the shell: mvn clean install -Dtest=FuraffinityRipperTest -Dgroups=slow mvn clean install -Dtest=FuraffinityRipperTest '-Dgroups=!slow' --- README.md | 5 ++++- pom.xml | 15 ++------------- .../tst/ripper/rippers/FuraffinityRipperTest.java | 3 +++ 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 63299485..5d31bc42 100644 --- a/README.md +++ b/README.md @@ -88,10 +88,13 @@ This will include all dependencies in the JAR. # Running Tests -After building you can run tests by running the following: +Tests can be marked as beeing slow or not. Default is to run all tests. Slow tests can be excluded to run, and slow +tests can be run on its own. After building you can run tests, quoting might be necessary depending on your shell: ```bash mvn test +mvn test -Dgroups=slow +mvn test '-Dgroups=!slow' ``` Please note that some tests may fail as sites change and our rippers become out of date. diff --git a/pom.xml b/pom.xml index 60e08138..7180ff27 100644 --- a/pom.xml +++ b/pom.xml @@ -143,18 +143,7 @@ maven-surefire-plugin - 2.22.2 - - - slow - - - - + 3.0.0-M5 @@ -163,7 +152,7 @@ org.apache.maven.plugins maven-surefire-report-plugin - 3.0.0-M3 + 3.0.0-M5 false diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java index 1efc30f7..8e2e359a 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java @@ -5,15 +5,18 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.FuraffinityRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class FuraffinityRipperTest extends RippersTest { @Test + @Tag("slow") public void testFuraffinityAlbum() throws IOException { FuraffinityRipper ripper = new FuraffinityRipper(new URL("https://www.furaffinity.net/gallery/spencerdragon/")); testRipper(ripper); } @Test + @Tag("slow") public void testFuraffinityScrap() throws IOException { FuraffinityRipper ripper = new FuraffinityRipper(new URL("http://www.furaffinity.net/scraps/sssonic2/")); testRipper(ripper); From ea5e761c793be8814d6e6c175388a3aaf3b1e774 Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 30 Jul 2020 02:58:53 +0200 Subject: [PATCH 069/512] RippersTest does not need a dummy test method --- .../com/rarchives/ripme/tst/ripper/rippers/RippersTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java index f1a95622..c09b8018 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java @@ -21,10 +21,6 @@ public class RippersTest { private final Logger logger = Logger.getLogger(RippersTest.class); - public void testStub() { - Assertions.assertTrue(true, "RippersTest must contain at lease one test."); - } - void testRipper(AbstractRipper ripper) { try { // Turn on Debug logging From 3a5e4d45d41265e5781ebc8f35364157d511e8fc Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 30 Jul 2020 08:53:21 +0200 Subject: [PATCH 070/512] disable second pichunter test which is executed and randomly failing when using tags --- .../rarchives/ripme/tst/ripper/rippers/PichunterRipperTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PichunterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PichunterRipperTest.java index a45f45aa..90a36bc7 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PichunterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PichunterRipperTest.java @@ -18,6 +18,7 @@ public class PichunterRipperTest extends RippersTest { } @Test + @Disabled("This test was commented out at 6/08/2018 because it was randomly failing due to issues with the site see https://github.com/RipMeApp/ripme/issues/867") public void testPichunterGalleryRip() throws IOException { // a photo set PichunterRipper ripper = new PichunterRipper( From a1ce03b505171fef0efe760523c19ef27fd7cb9d Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 25 Oct 2020 12:36:22 +0100 Subject: [PATCH 071/512] allow to switch off tests which do not run reliable, shaky --- pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pom.xml b/pom.xml index 7180ff27..5ccd68bd 100644 --- a/pom.xml +++ b/pom.xml @@ -144,6 +144,9 @@ maven-surefire-plugin 3.0.0-M5 + + shaky + From cbdf457419c46b141f651c54a6b82268383a2bb2 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 25 Oct 2020 12:58:02 +0100 Subject: [PATCH 072/512] disable broken tests, tag as flaky which sometimes fail --- README.md | 7 ++++--- pom.xml | 2 +- .../ripme/tst/ripper/rippers/CheveretoRipperTest.java | 2 ++ .../ripme/tst/ripper/rippers/ErotivRipperTest.java | 2 ++ .../ripme/tst/ripper/rippers/FolioRipperTest.java | 2 ++ .../ripme/tst/ripper/rippers/FooktubeRipperTest.java | 2 ++ .../ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java | 2 ++ .../ripme/tst/ripper/rippers/ImgurRipperTest.java | 2 ++ .../ripme/tst/ripper/rippers/LusciousRipperTest.java | 3 +++ .../ripme/tst/ripper/rippers/PichunterRipperTest.java | 6 +++--- .../ripme/tst/ripper/rippers/RedgifsRipperTest.java | 2 ++ 11 files changed, 25 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 5d31bc42..e5d5146c 100644 --- a/README.md +++ b/README.md @@ -88,12 +88,13 @@ This will include all dependencies in the JAR. # Running Tests -Tests can be marked as beeing slow or not. Default is to run all tests. Slow tests can be excluded to run, and slow -tests can be run on its own. After building you can run tests, quoting might be necessary depending on your shell: +Tests can be marked as beeing slow, or flaky. Default is to run all but the flaky tests. Slow tests can be excluded to +run. slow and flaky tests can be run on its own. After building you can run tests, quoting might be necessary depending +on your shell: ```bash mvn test -mvn test -Dgroups=slow +mvn test -Dgroups=flaky,slow mvn test '-Dgroups=!slow' ``` diff --git a/pom.xml b/pom.xml index 5ccd68bd..88dc0b58 100644 --- a/pom.xml +++ b/pom.xml @@ -145,7 +145,7 @@ maven-surefire-plugin 3.0.0-M5 - shaky + flaky diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java index 385464da..420fcb00 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.CheveretoRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class CheveretoRipperTest extends RippersTest { @@ -13,6 +14,7 @@ public class CheveretoRipperTest extends RippersTest { testRipper(ripper); } @Test + @Tag("flaky") public void testSubdirAlbum() throws IOException { CheveretoRipper ripper = new CheveretoRipper(new URL("https://kenzato.uk/booru/album/TnEc")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErotivRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErotivRipperTest.java index 432b7fb9..b4afdd67 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErotivRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErotivRipperTest.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ErotivRipper; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class ErotivRipperTest extends RippersTest { @@ -21,6 +22,7 @@ public class ErotivRipperTest extends RippersTest { } @Test + @Disabled("test or ripper broken") public void testGetURLsFromPage() throws IOException { URL url = new URL("https://erotiv.io/e/1568314255"); ErotivRipper ripper = new ErotivRipper(url); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java index 3a8627bd..9384aebf 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java @@ -5,6 +5,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.FolioRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class FolioRipperTest extends RippersTest { @@ -13,6 +14,7 @@ public class FolioRipperTest extends RippersTest { * @throws IOException */ @Test + @Disabled("test or ripper broken") public void testFolioRip() throws IOException { FolioRipper ripper = new FolioRipper(new URL("https://folio.ink/DmBe6i")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FooktubeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FooktubeRipperTest.java index b98cabdd..10131c19 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FooktubeRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FooktubeRipperTest.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.FooktubeRipper; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class FooktubeRipperTest extends RippersTest { @Test + @Disabled("test or ripper broken") public void testFooktubeVideo() throws IOException { FooktubeRipper ripper = new FooktubeRipper(new URL("https://fooktube.com/video/641/in-the-cinema")); //pick any video from the front page testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java index 18068b18..5b8c4558 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java @@ -5,10 +5,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.GfycatporntubeRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class GfycatporntubeRipperTest extends RippersTest { @Test + @Tag("flaky") public void testRip() throws IOException { GfycatporntubeRipper ripper = new GfycatporntubeRipper(new URL("https://gfycatporntube.com/blowjob-bunny-puts-on-a-show/")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java index 3772d5f2..37ef50eb 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java @@ -5,6 +5,7 @@ import com.rarchives.ripme.ripper.rippers.ImgurRipper.ImgurAlbum; import com.rarchives.ripme.utils.RipUtils; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -57,6 +58,7 @@ public class ImgurRipperTest extends RippersTest { } @Test + @Disabled("test or ripper broken") public void testImgurSingleImage() throws IOException { List contentURLs = new ArrayList<>(); contentURLs.add(new URL("http://imgur.com/qbfcLyG")); // Single image URL diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java index 52c4a3c5..7c797b4b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java @@ -5,10 +5,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.LusciousRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class LusciousRipperTest extends RippersTest { @Test + @Disabled("test or ripper broken") public void testPahealRipper() throws IOException { // a photo set LusciousRipper ripper = new LusciousRipper( @@ -24,6 +26,7 @@ public class LusciousRipperTest extends RippersTest { } @Test + @Disabled("test or ripper broken") public void testGetNextPage() throws IOException { URL multiPageAlbumUrl = new URL("https://luscious.net/albums/women-of-color_58/"); LusciousRipper multiPageRipper = new LusciousRipper(multiPageAlbumUrl); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PichunterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PichunterRipperTest.java index 90a36bc7..9ba9110b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PichunterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PichunterRipperTest.java @@ -5,12 +5,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.PichunterRipper; -import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class PichunterRipperTest extends RippersTest { @Test - @Disabled("This test was commented out at 6/08/2018 because it was randomly failing due to issues with the site see https://github.com/RipMeApp/ripme/issues/867") + @Tag("flaky") public void testPichunterModelPageRip() throws IOException { // A non-photoset PichunterRipper ripper = new PichunterRipper(new URL("https://www.pichunter.com/models/Madison_Ivy")); @@ -18,7 +18,7 @@ public class PichunterRipperTest extends RippersTest { } @Test - @Disabled("This test was commented out at 6/08/2018 because it was randomly failing due to issues with the site see https://github.com/RipMeApp/ripme/issues/867") + @Tag("flaky") public void testPichunterGalleryRip() throws IOException { // a photo set PichunterRipper ripper = new PichunterRipper( diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java index bcca515c..9789417d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java @@ -14,6 +14,7 @@ public class RedgifsRipperTest extends RippersTest { * @throws IOException */ @Test + @Disabled("test or ripper broken") public void testRedgifsGoodURL() throws IOException{ RedgifsRipper ripper = new RedgifsRipper(new URL("https://www.redgifs.com/watch/talkativewarpeddragon-petite")); testRipper(ripper); @@ -24,6 +25,7 @@ public class RedgifsRipperTest extends RippersTest { * @throws IOException */ @Test + @Tag("flaky") public void testRedgifsBadRL() throws IOException{ RedgifsRipper ripper = new RedgifsRipper(new URL("https://www.gifdeliverynetwork.com/foolishelasticchimpanzee")); testRipper(ripper); From ab4b584a97497582b5fbbd14d1a815ed08b55453 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 25 Oct 2020 15:09:15 +0100 Subject: [PATCH 073/512] reddit flaky tests --- .../ripme/tst/ripper/rippers/RedditRipperTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java index d5d9600d..2d0c6a58 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java @@ -7,19 +7,20 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.RedditRipper; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class RedditRipperTest extends RippersTest { @Test - @Disabled("Rip is flaky") // https://github.com/RipMeApp/ripme/issues/253 + @Tag("flaky") // https://github.com/RipMeApp/ripme/issues/253 public void testRedditSubredditRip() throws IOException { RedditRipper ripper = new RedditRipper(new URL("http://www.reddit.com/r/nsfw_oc")); testRipper(ripper); } @Test - @Disabled("Rip is flaky") // https://github.com/RipMeApp/ripme/issues/253 + @Tag("flaky") // https://github.com/RipMeApp/ripme/issues/253 public void testRedditSubredditTopRip() throws IOException { RedditRipper ripper = new RedditRipper(new URL("http://www.reddit.com/r/nsfw_oc/top?t=all")); testRipper(ripper); @@ -33,12 +34,13 @@ public class RedditRipperTest extends RippersTest { testRipper(ripper); } - /** + /**testRedditSubredditRip:19 * GFYCAT TEST Tests a good GfycatURL (no "/gifs/detail") * * @throws IOException */ @Test + @Tag("flaky") public void testRedditGfyGoodURL() throws IOException { RedditRipper ripper = new RedditRipper( new URL("https://www.reddit.com/r/bottesting/comments/7msozf/good_link/")); @@ -51,6 +53,7 @@ public class RedditRipperTest extends RippersTest { * @throws IOException */ @Test + @Tag("flaky") public void testRedditGfyBadURL() throws IOException { RedditRipper ripper = new RedditRipper( new URL("https://www.reddit.com/r/bottesting/comments/7msmhi/bad_link/")); From 122b6d1d5e64b72679d1ffd8bec76fb3b111b663 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 25 Oct 2020 20:05:14 +0100 Subject: [PATCH 074/512] imagefapripertest, xhamster flaky --- .../rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java | 2 ++ .../rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java index 6a112a5d..0ed0add2 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java @@ -7,10 +7,12 @@ import java.util.Map; import com.rarchives.ripme.ripper.rippers.ImagefapRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class ImagefapRipperTest extends RippersTest { @Test + @Tag("flaky") public void testImagefapAlbums() throws IOException { Map testURLs = new HashMap<>(); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java index 27b8364a..a5f1beaa 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java @@ -7,6 +7,7 @@ import com.rarchives.ripme.ripper.rippers.XhamsterRipper; import org.jsoup.nodes.Document; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -32,6 +33,7 @@ public class XhamsterRipperTest extends RippersTest { testRipper(ripper); } @Test + @Tag("flaky") public void testXhamsterAlbumDesiDomain() throws IOException { XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.desi/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664")); testRipper(ripper); From 018e782b2eae410d7a7a4f21924fcd892a5d9639 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 25 Oct 2020 15:27:56 +0100 Subject: [PATCH 075/512] java-14 in test --- .github/workflows/maven.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 93719005..bb44b0c8 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -9,11 +9,11 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - java: [1.8, 1.9] + java: [1.8, 1.14] steps: - uses: actions/checkout@v1 - - name: Set up JDK 1.8 + - name: Set up JDK uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} From 61c8bce13cffdd1ed0afcc301df2210f080be08a Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 25 Oct 2020 21:41:27 +0100 Subject: [PATCH 076/512] twitter test flaky --- .../rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java index 788808c8..8e746c9e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java @@ -5,16 +5,19 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.TwitterRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class TwitterRipperTest extends RippersTest { @Test + @Tag("flaky") public void testTwitterUserRip() throws IOException { TwitterRipper ripper = new TwitterRipper(new URL("https://twitter.com/danngamber01/media")); testRipper(ripper); } @Test + @Tag("flaky") public void testTwitterSearchRip() throws IOException { TwitterRipper ripper = new TwitterRipper( new URL("https://twitter.com/search?f=tweets&q=from%3Aalinalixxx%20filter%3Aimages&src=typd")); From 8ec221aa525879cb0be2e7cfcc7593be7b27769e Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 26 Oct 2020 01:42:25 +0100 Subject: [PATCH 077/512] allow to set excludedGroups on command line --- README.md | 2 +- pom.xml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e5d5146c..8ff095f1 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ on your shell: ```bash mvn test -mvn test -Dgroups=flaky,slow +mvn test -DexcludedGroups= -Dgroups=flaky,slow mvn test '-Dgroups=!slow' ``` diff --git a/pom.xml b/pom.xml index 88dc0b58..9604b0fe 100644 --- a/pom.xml +++ b/pom.xml @@ -7,6 +7,7 @@ ripme http://rip.rarchives.com + flaky UTF-8 @@ -145,7 +146,7 @@ maven-surefire-plugin 3.0.0-M5 - flaky + ${excludedGroups} From 0720a2d8e340d262d381e99a32a8dacab2777225 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 26 Oct 2020 03:42:47 +0100 Subject: [PATCH 078/512] theyiffgalery flaky --- .../ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java index ce8ce88c..3c9b6a4a 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java @@ -5,10 +5,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.TheyiffgalleryRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class TheyiffgalleryRipperTest extends RippersTest { @Test + @Tag("flaky") public void testTheyiffgallery() throws IOException { TheyiffgalleryRipper ripper = new TheyiffgalleryRipper(new URL("https://theyiffgallery.com/index?/category/4303")); testRipper(ripper); From 04b5c6afb6f994d890635e2f7c221b4be14fc7ed Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 26 Oct 2020 08:49:12 +0100 Subject: [PATCH 079/512] flaky instead of disableld in WordpressComicRipperTest --- .../ripper/rippers/WordpressComicRipperTest.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java index cad697de..6a647286 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java @@ -6,7 +6,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.WordpressComicRipper; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class WordpressComicRipperTest extends RippersTest { @@ -23,7 +23,7 @@ public class WordpressComicRipperTest extends RippersTest { // http://shipinbottle.pepsaga.com/?p=281 @Test - @Disabled("https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI") + @Tag("flaky") // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI public void test_totempole666() throws IOException { WordpressComicRipper ripper = new WordpressComicRipper( new URL("http://www.totempole666.com/comic/first-time-for-everything-00-cover/")); @@ -31,14 +31,14 @@ public class WordpressComicRipperTest extends RippersTest { } @Test - @Disabled("https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI") + @Tag("flaky") // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI public void test_buttsmithy() throws IOException { WordpressComicRipper ripper = new WordpressComicRipper(new URL("http://buttsmithy.com/archives/comic/p1")); testRipper(ripper); } @Test - @Disabled("https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI") + @Tag("flaky") // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI public void test_themonsterunderthebed() throws IOException { WordpressComicRipper ripper = new WordpressComicRipper( new URL("http://themonsterunderthebed.net/?comic=test-post")); @@ -72,13 +72,14 @@ public class WordpressComicRipperTest extends RippersTest { } @Test - @Disabled("https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI") + @Tag("flaky") // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI public void test_freeadultcomix() throws IOException { WordpressComicRipper ripper = new WordpressComicRipper( new URL("http://freeadultcomix.com/finders-feepaid-in-full-sparrow/")); testRipper(ripper); } @Test + @Tag("flaky") public void test_delvecomic() throws IOException { WordpressComicRipper ripper = new WordpressComicRipper( new URL("http://thisis.delvecomic.com/NewWP/comic/in-too-deep/")); @@ -110,7 +111,7 @@ public class WordpressComicRipperTest extends RippersTest { } @Test - @Disabled("https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI") + @Tag("flaky") // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI public void test_pepsaga() throws IOException { WordpressComicRipper ripper = new WordpressComicRipper(new URL("http://shipinbottle.pepsaga.com/?p=281")); testRipper(ripper); From 5ffabbe4ba1f2b0e88881544aa581a02efd74f46 Mon Sep 17 00:00:00 2001 From: Frankenst1 <73558058+Frankenst1@users.noreply.github.com> Date: Tue, 27 Oct 2020 23:25:00 +0100 Subject: [PATCH 080/512] Fix imagefap ripper to take rate limit into account. --- .../ripme/ripper/rippers/ImagefapRipper.java | 94 ++++++++++++++++++- 1 file changed, 90 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java index b1a27b47..14d21aa9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java @@ -12,6 +12,7 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; public class ImagefapRipper extends AbstractHTMLRipper { @@ -19,6 +20,18 @@ public class ImagefapRipper extends AbstractHTMLRipper { private Document albumDoc = null; private boolean isNewAlbumType = false; + private int callsMade = 0; + private long startTime = System.nanoTime(); + + private static final int RETRY_LIMIT = 10; + private static final int RATE_LIMIT_HOUR = 1000; + + // All sleep times are in milliseconds + private static final int PAGE_SLEEP_TIME = 60 * 60 * 1000 / RATE_LIMIT_HOUR; + private static final int IMAGE_SLEEP_TIME = 60 * 60 * 1000 / RATE_LIMIT_HOUR; + // Timeout when blocked = 1 hours. Retry every retry within the hour mark + 1 time after the hour mark. + private static final int IP_BLOCK_SLEEP_TIME = (int) Math.round((double) 60 / (RETRY_LIMIT - 1) * 60 * 1000); + public ImagefapRipper(URL url) throws IOException { super(url); } @@ -97,7 +110,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { if (albumDoc == null) { - albumDoc = Http.url(url).get(); + albumDoc = getPageWithRetries(url); } return albumDoc; } @@ -114,8 +127,13 @@ public class ImagefapRipper extends AbstractHTMLRipper { if (nextURL == null) { throw new IOException("No next page found"); } - sleep(1000); - return Http.url(nextURL).get(); + // Sleep before fetching next page. + sleep(PAGE_SLEEP_TIME); + + // Load next page + Document nextPage = getPageWithRetries(new URL(nextURL)); + + return nextPage; } @Override @@ -156,11 +174,79 @@ public class ImagefapRipper extends AbstractHTMLRipper { private String getFullSizedImage(String pageURL) { try { - Document doc = Http.url(pageURL).get(); + // Sleep before fetching image. + sleep(IMAGE_SLEEP_TIME); + + Document doc = getPageWithRetries(new URL(pageURL)); return doc.select("img#mainPhoto").attr("src"); } catch (IOException e) { return null; } } + /** + * Attempts to get page, checks for IP ban, waits. + * @param url + * @return Page document + * @throws IOException If page loading errors, or if retries are exhausted + */ + private Document getPageWithRetries(URL url) throws IOException { + Document doc; + int retries = RETRY_LIMIT; + while (true) { + sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm()); + + // For debugging rate limit checker. Useful to track wheter the timeout should be altered or not. + callsMade++; + checkRateLimit(); + + LOGGER.info("Retrieving " + url); + doc = Http.url(url) + .get(); + + + if (doc.toString().contains("Your IP made too many requests to our servers and we need to check that you are a real human being")) { + if (retries == 0) { + throw new IOException("Hit rate limit and maximum number of retries, giving up"); + } + String message = "Hit rate limit while loading " + url + ", sleeping for " + IP_BLOCK_SLEEP_TIME + "ms, " + retries + " retries remaining"; + LOGGER.warn(message); + sendUpdate(STATUS.DOWNLOAD_WARN, message); + retries--; + try { + Thread.sleep(IP_BLOCK_SLEEP_TIME); + } catch (InterruptedException e) { + throw new IOException("Interrupted while waiting for rate limit to subside"); + } + } + else { + return doc; + } + } + } + + /** + * Used for debugging the rate limit issue. + * This in order to prevent hitting the rate limit altoghether by remaining under the limit threshold. + * @return Long duration + */ + private long checkRateLimit() { + long endTime = System.nanoTime(); + long duration = (endTime - startTime) / 1000000; + + int rateLimitMinute = 100; + int rateLimitFiveMinutes = 200; + int rateLimitHour = RATE_LIMIT_HOUR; // Request allowed every 3.6 seconds. + + if(duration / 1000 < 60){ + LOGGER.debug("Rate limit: " + (rateLimitMinute - callsMade) + " calls remaining for first minute mark."); + } else if(duration / 1000 < 300){ + LOGGER.debug("Rate limit: " + (rateLimitFiveMinutes - callsMade) + " calls remaining for first 5 minute mark."); + } else if(duration / 1000 < 3600){ + LOGGER.debug("Rate limit: " + (RATE_LIMIT_HOUR - callsMade) + " calls remaining for first hour mark."); + } + + return duration; + } + } From 2544b20ebc2744fc5cd9fdff266fe94695bae65f Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 21 Nov 2020 09:11:58 +0100 Subject: [PATCH 081/512] src --> data-src in AllporncomicRipper --- .../com/rarchives/ripme/ripper/rippers/AllporncomicRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/AllporncomicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/AllporncomicRipper.java index 66ddba4d..a5fbbd0f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/AllporncomicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/AllporncomicRipper.java @@ -56,7 +56,7 @@ public class AllporncomicRipper extends AbstractHTMLRipper { public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); for (Element el : doc.select(".wp-manga-chapter-img")) { - result.add(el.attr("src")); + result.add(el.attr("data-src")); } return result; } From 19ea20dcd62d6ea1197073ce9a7236c5c1f47556 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 22 Nov 2020 02:08:59 -0500 Subject: [PATCH 082/512] 1.7.94: Added reddit gallery support; Fixed AllporncomicRipper; Fix imagefap ripper; instagramRipper, replaced Nashorn with GraalVM.js --- pom.xml | 2 +- ripme.json | 5 +++-- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 9604b0fe..fb1bb42e 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.rarchives.ripme ripme jar - 1.7.93 + 1.7.94 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 1c7cd314..79030d5e 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "currentHash": "357639535c112ca857117151321141cb08ad313021079d522c2c5d15ee799998", + "currentHash": "1ebe144d0f294c4e8b482742a2588201f8b962fbcebbbd109627e1fc9e2b6d94", "changeList": [ + "1.7.94: Added reddit gallery support; Fixed AllporncomicRipper; Fix imagefap ripper; instagramRipper, replaced Nashorn with GraalVM.js", "1.7.93: Fixed Motherless ripper; Fixed e621 ripper; Updated pt_PT translation; Implemented redgifs Ripper; added missing translation to Korean/KR; Fixed elecx ripper; Added ripper for HentaiNexus", "1.7.92: Added read-comic.com ripper; Fix Pawoo ripper; Add ChineseSimplified language file; Fixed artstation ripper", "1.7.91: Fixed luscious ripper. Fixed VK ripper; Added Kingcomix ripper", @@ -265,5 +266,5 @@ "1.0.3: Added VK.com ripper", "1.0.1: Added auto-update functionality" ], - "latestVersion": "1.7.93" + "latestVersion": "1.7.94" } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 88dc6a17..8c1b415e 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -23,7 +23,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.93"; + private static final String DEFAULT_VERSION = "1.7.94"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static String mainFileName; From df2b5463915b5f9dbc358b5f7948738e92a3c57d Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 23 Nov 2020 00:35:15 -0500 Subject: [PATCH 083/512] Added porncomixinfo.net ripper and test --- .../ripper/rippers/PorncomixinfoRipper.java | 85 +++++++++++++++++++ .../rippers/PorncomixinfoRipperTest.java | 15 ++++ 2 files changed, 100 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixinfoRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixinfoRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixinfoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixinfoRipper.java new file mode 100644 index 00000000..241ad5d7 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixinfoRipper.java @@ -0,0 +1,85 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +public class PorncomixinfoRipper extends AbstractHTMLRipper { + + public PorncomixinfoRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "porncomixinfo"; + } + + @Override + public String getDomain() { + return "porncomixinfo.net"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https://porncomixinfo.net/chapter/([a-zA-Z1-9_-]*)/([a-zA-Z1-9_-]*)/?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected porncomixinfo URL format: " + + "porncomixinfo.net/chapter/CHAP/ID - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public Document getNextPage(Document doc) throws IOException { + // Find next page + String nextUrl = ""; + // We use comic-nav-next to the find the next page + Element elem = doc.select("a.next_page").first(); + if (elem == null) { + throw new IOException("No more pages"); + } + String nextPage = elem.attr("href"); + // Some times this returns a empty string + // This for stops that + if (nextPage.equals("")) { + return null; + } + else { + return Http.url(nextPage).get(); + } + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + for (Element el : doc.select("img.wp-manga-chapter-img")) { { + String imageSource = el.attr("src"); + result.add(imageSource); + } + } + return result; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixinfoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixinfoRipperTest.java new file mode 100644 index 00000000..6a839036 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixinfoRipperTest.java @@ -0,0 +1,15 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.PorncomixinfoRipper; +import org.junit.Test; + +public class PorncomixinfoRipperTest extends RippersTest { + @Test + public void testRip() throws IOException { + PorncomixinfoRipper ripper = new PorncomixinfoRipper(new URL("https://porncomixinfo.net/chapter/alx-come-to-naught-down-in-flames-up-in-smoke-tracy-scops/alx-come-to-naught-down-in-flames-up-in-smoke-tracy-scops/")); + testRipper(ripper); + } +} From 0d8380b99285ddde2c37d294c1439a4ff32a2d87 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Wed, 25 Nov 2020 19:09:07 -0500 Subject: [PATCH 084/512] Fixed issue which caused ripme to fail to rip more than 1 url when running in the background --- .../java/com/rarchives/ripme/ui/MainWindow.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index bf19a7be..92b8071c 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1279,11 +1279,15 @@ public final class MainWindow implements Runnable, RipStatusHandler { ripper.setObserver(this); Thread t = new Thread(ripper); if (configShowPopup.isSelected() && (!mainFrame.isVisible() || !mainFrame.isActive())) { - mainFrame.toFront(); - mainFrame.setAlwaysOnTop(true); - trayIcon.displayMessage(mainFrame.getTitle(), "Started ripping " + ripper.getURL().toExternalForm(), - MessageType.INFO); - mainFrame.setAlwaysOnTop(false); + try { + mainFrame.toFront(); + mainFrame.setAlwaysOnTop(true); + trayIcon.displayMessage(mainFrame.getTitle(), "Started ripping " + ripper.getURL().toExternalForm(), + MessageType.INFO); + mainFrame.setAlwaysOnTop(false); + } catch (NullPointerException e) { + LOGGER.error("Could not send popup, are tray icons supported?"); + } } return t; } catch (Exception e) { From 728bfa16eac1ec241682762fd73e834f208f7ae5 Mon Sep 17 00:00:00 2001 From: Omar Morales Date: Tue, 8 Dec 2020 10:43:08 -0600 Subject: [PATCH 085/512] NullPointerException handled for invalid save path. #1785 --- .../com/rarchives/ripme/ripper/DownloadFileThread.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index ba1104eb..c05fe0f9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -282,7 +282,14 @@ class DownloadFileThread extends Thread { logger.debug("IOException", e); logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + e.getMessage()); - } finally { + } catch (NullPointerException npe){ + + logger.error("[!] " + Utils.getLocalizedString("failed.to.download") + " for URL " + url); + observer.downloadErrored(url, + Utils.getLocalizedString("failed.to.download") + " " + url.toExternalForm()); + return; + + }finally { // Close any open streams try { if (bis != null) { From d1fa53dc8c04e2762bf9f864489772e6f5b0f260 Mon Sep 17 00:00:00 2001 From: PaaaulZ <46759927+PaaaulZ@users.noreply.github.com> Date: Thu, 17 Dec 2020 01:45:41 +0100 Subject: [PATCH 086/512] Added support for scrolller.com No support for the "sorting" parameter --- .../ripme/ripper/rippers/ScrolllerRipper.java | 213 ++++++++++++++++++ .../ripper/rippers/ScrolllerRipperTest.java | 55 +++++ 2 files changed, 268 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java new file mode 100644 index 00000000..afadcd1f --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java @@ -0,0 +1,213 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import com.rarchives.ripme.ripper.AbstractJSONRipper; + +public class ScrolllerRipper extends AbstractJSONRipper { + + public ScrolllerRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "scrolller"; + } + @Override + public String getDomain() { + return "scrolller.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + // Typical URL is: https://scrolller.com/r/subreddit + // Parameters like "filter" and "sort" can be passed (ex: https://scrolller.com/r/subreddit?filter=xxx&sort=yyyy) + Pattern p = Pattern.compile("^https?://scrolller\\.com/r/([a-zA-Z0-9]+).*?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected scrolller.com URL format: " + + "scrolller.com/r/subreddit OR scroller.com/r/subreddit?filter= - got " + url + "instead"); + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + + + private JSONObject prepareQuery(String iterator, String gid) throws IOException { + + // Prepares the JSONObject we need to pass to the GraphQL query. + + String queryString = "query SubredditQuery( $url: String! $filter: SubredditPostFilter $iterator: String ) { getSubreddit(url: $url) { children( limit: 50 iterator: $iterator filter: $filter ) { iterator items { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } } } }"; + String filterString = convertFilterString(getParameter(this.url,"filter")); + + JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid)); + + if (iterator != null) { + // Iterator is not present on the first page + variablesObject.put("iterator", iterator); + } + if (!filterString.equals("NOFILTER")) { + // We could also pass filter="" but not including it if not present is cleaner + variablesObject.put("filter", filterString); + } + + JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", queryString); + + return getPosts(finalQueryObject); + + } + + + public String convertFilterString(String filterParameter) { + // Converts the ?filter= parameter of the URL to one that can be used in the GraphQL query + // I could basically remove the last "s" and uppercase instead of this switch statement but this looks easier to read. + switch (filterParameter) { + case "pictures": + return "PICTURE"; + case "videos": + return "VIDEO"; + case "albums": + return "ALBUM"; + case "": + return "NOFILTER"; + default: + LOGGER.error(String.format("Invalid filter %s using no filter",filterParameter)); + return ""; + } + } + + public String getParameter(URL url, String parameter) throws MalformedURLException { + // Gets passed parameters from the URL + String toReplace = String.format("https://scrolller.com/r/%s?",getGID(url)); + List args= URLEncodedUtils.parse(url.toExternalForm(), Charset.defaultCharset()); + for (NameValuePair arg:args) { + // First parameter contains part of the url so we have to remove it + // Ex: for the url https://scrolller.com/r/CatsStandingUp?filter=xxxx&sort=yyyy + // 1) arg.getName() => https://scrolller.com/r/CatsStandingUp?filter + // 2) arg.getName() => sort + + if (arg.getName().replace(toReplace,"").equals((parameter))) { + return arg.getValue(); + } + } + return ""; + } + + private JSONObject getPosts(JSONObject data) { + // The actual GraphQL query call + + // JSoup wants POST data in key=value but I need to write a JSON body so I can't use it... + try { + + String url = "https://api.scrolller.com/api/v2/graphql"; + + URL obj = new URL(url); + HttpURLConnection conn = (HttpURLConnection) obj.openConnection(); + conn.setReadTimeout(5000); + conn.addRequestProperty("Accept-Language", "en-US,en;q=0.8"); + conn.addRequestProperty("User-Agent", "Mozilla"); + conn.addRequestProperty("Referer", "scrolller.com"); + + conn.setDoOutput(true); + + OutputStreamWriter w = new OutputStreamWriter(conn.getOutputStream(), "UTF-8"); + + w.write(data.toString()); + w.close(); + + BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream())); + String inputLine; + StringBuffer html = new StringBuffer(); + + while ((inputLine = in.readLine()) != null) { + html.append(inputLine); + } + + in.close(); + conn.disconnect(); + + return new JSONObject(html.toString()); + + } catch (Exception e) { + e.printStackTrace(); + } + + return new JSONObject("{}"); +} + + + @Override + protected List getURLsFromJSON(JSONObject json) throws JSONException { + JSONArray itemsList = json.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").getJSONArray("items"); + int bestArea = 0; + String bestUrl = ""; + List list = new ArrayList<>(); + + + for (Object item : itemsList) { + JSONArray sourcesTMP = ((JSONObject) item).getJSONArray("mediaSources"); + for (Object sourceTMP : sourcesTMP) + { + int widthTMP = ((JSONObject) sourceTMP).getInt("width"); + int heightTMP = ((JSONObject) sourceTMP).getInt("height"); + int areaTMP = widthTMP * heightTMP; + + if (areaTMP > bestArea) { + // Better way to determine best image? + bestArea = widthTMP; + bestUrl = ((JSONObject) sourceTMP).getString("url"); + } + } + list.add(bestUrl); + bestUrl = ""; + bestArea = 0; + } + return list; + } + + @Override + protected JSONObject getFirstPage() throws IOException { + if (getParameter(url,"sort") != null) { + // I need support for the WebSocket protocol to implement sorting. + // A GraphQL query to the API with the "sortBy" variable can't come from a POST request or it will return error 500, it has to come from a WebSocket. + LOGGER.warn("Sorting is not currently implemented and it will be ignored"); + } + return prepareQuery(null, this.getGID(url)); + } + + @Override + public JSONObject getNextPage(JSONObject source) throws IOException { + // Every call the the API contains an "iterator" string that we need to pass to the API to get the next page + // Checking if iterator is null is not working for some reason, hence why the weird "iterator.toString().equals("null")" + Object iterator = source.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").get("iterator"); + if (!iterator.toString().equals("null")) { + return prepareQuery(iterator.toString(), this.getGID(url)); + } else { + return null; + } + + } + +} \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java new file mode 100644 index 00000000..feef282b --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java @@ -0,0 +1,55 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.ScrolllerRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; + +public class ScrolllerRipperTest extends RippersTest { + /*@Test + public void testScrolllerGID() throws IOException { + Map testURLs = new HashMap<>(); + + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp"), "CatsStandingUp"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=pictures"), "CatsStandingUp"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?sort=top&filter=pictures"), "CatsStandingUp"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=pictures&sort=top"), "CatsStandingUp"); + for (URL url : testURLs.keySet()) { + ScrolllerRipper ripper = new ScrolllerRipper(url); + ripper.setup(); + System.out.println(testURLs.get(url) + " |=>| " + ripper.getGID(ripper.getURL())); + Assertions.assertEquals(testURLs.get(url), ripper.getGID(ripper.getURL())); + deleteDir(ripper.getWorkingDir()); + } + }*/ + + @Test + public void testScrolllerFilterRegex() throws IOException { + Map testURLs = new HashMap<>(); + + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp"), ""); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=pictures"), "PICTURE"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=videos"), "VIDEO"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=albums"), "ALBUM"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?sort=top&filter=pictures"), "PICTURE"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?sort=top&filter=videos"), "VIDEO"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?sort=top&filter=albums"), "ALBUM"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=pictures&sort=top"), "PICTURE"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=videos&sort=top"), "VIDEO"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=albums&sort=top"), "ALBUM"); + for (URL url : testURLs.keySet()) { + ScrolllerRipper ripper = new ScrolllerRipper(url); + ripper.setup(); + System.out.println(url + " ==> " + testURLs.get(url) + " => " + ripper.convertFilterString(ripper.getParameter(ripper.getURL(),"filter"))); + Assertions.assertEquals(testURLs.get(url), ripper.convertFilterString(ripper.getParameter(ripper.getURL(),"filter"))); + deleteDir(ripper.getWorkingDir()); + } + } + + + +} From 0dded85ddd5a16a0bf32020b0693d0d2298e9088 Mon Sep 17 00:00:00 2001 From: PaaaulZ <46759927+PaaaulZ@users.noreply.github.com> Date: Fri, 18 Dec 2020 22:17:41 +0100 Subject: [PATCH 087/512] Fixed ripper for HentaiNexus --- .../ripper/rippers/HentaiNexusRipper.java | 216 +++++++++++------- .../ripper/rippers/HentainexusRipperTest.java | 35 ++- 2 files changed, 167 insertions(+), 84 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java index 56ce0d2f..ca709418 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java @@ -4,27 +4,22 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.rarchives.ripme.utils.Http; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import com.rarchives.ripme.ripper.AbstractJSONRipper; +import org.jsoup.nodes.DataNode; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; -import com.rarchives.ripme.utils.Http; -import com.rarchives.ripme.utils.Utils; - -public class HentaiNexusRipper extends AbstractHTMLRipper { - - private Document firstPage; - private DownloadThreadPool hentainexusThreadPool = new DownloadThreadPool("hentainexus"); - @Override - public DownloadThreadPool getThreadPool() { - return hentainexusThreadPool; - } +public class HentaiNexusRipper extends AbstractJSONRipper { public HentaiNexusRipper(URL url) throws IOException { super(url); @@ -34,7 +29,6 @@ public class HentaiNexusRipper extends AbstractHTMLRipper { public String getHost() { return "hentainexus"; } - @Override public String getDomain() { return "hentainexus.com"; @@ -42,88 +36,148 @@ public class HentaiNexusRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("https?://hentainexus\\.com/view/([a-zA-Z0-9_\\-%]*)/?$"); + /* + Valid URLs are /view/id, /read/id and those 2 with #pagenumber + https://hentainexus.com/view/9202 + https://hentainexus.com/read/9202 + https://hentainexus.com/view/9202#001 + https://hentainexus.com/read/9202#001 + */ + + Pattern p = Pattern.compile("^https?://hentainexus\\.com/(?:view|read)/([0-9]+)(?:\\#[0-9]+)*$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); } throw new MalformedURLException("Expected hentainexus.com URL format: " + - "hentainexus.com/view/NUMBER - got " + url + " instead"); - } - - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - if (firstPage == null) { - firstPage = Http.url(url).get(); - } - return firstPage; - } - - @Override - public List getURLsFromPage(Document doc) { - List imageURLs = new ArrayList<>(); - Elements thumbs = doc.select("div.is-multiline > div.column > a"); - for (Element el : thumbs) { - imageURLs.add("https://" + getDomain() + el.attr("href")); - } - return imageURLs; - } - - @Override - public String getAlbumTitle(URL url) throws MalformedURLException { - try { - Document gallery = Http.url(url).get(); - return getHost() + "_" + gallery.select("h1.title").text(); - } catch (IOException e) { - LOGGER.info("Falling back"); - } - - return super.getAlbumTitle(url); + "hentainexus.com/view/id OR hentainexus.com/read/id - got " + url + "instead"); } @Override public void downloadURL(URL url, int index) { - HentaiNexusImageThread t = new HentaiNexusImageThread(url, index); - hentainexusThreadPool.addThread(t); + addURLToDownload(url, getPrefix(index)); } - /** - * Helper class to find and download images found on "image" pages - */ - private class HentaiNexusImageThread extends Thread { - private URL url; - private int index; - HentaiNexusImageThread(URL url, int index) { - super(); - this.url = url; - this.index = index; + @Override + protected List getURLsFromJSON(JSONObject json) throws JSONException { + + List urlList = new ArrayList<>(); + + JSONArray imagesList = json.getJSONArray("f"); + String host = json.getString("b"); + String folder = json.getString("r"); + String id = json.getString("i"); + + for (Object singleImage : imagesList) { + String hashTMP = ((JSONObject) singleImage).getString("h"); + String fileNameTMP = ((JSONObject) singleImage).getString("p"); + String imageUrlTMP = String.format("%s%s%s/%s/%s",host,folder,hashTMP,id,fileNameTMP); + urlList.add(imageUrlTMP); } - @Override - public void run() { - fetchImage(); - } + return urlList; + } - private void fetchImage() { - try { - Document doc = Http.url(url).retries(3).get(); - Elements images = doc.select("figure.image > img"); - if (images.isEmpty()) { - LOGGER.warn("Image not found at " + this.url); - return; + @Override + protected JSONObject getFirstPage() throws IOException { + String jsonEncodedString = getJsonEncodedStringFromPage(); + String jsonDecodedString = decodeJsonString(jsonEncodedString); + return new JSONObject(jsonDecodedString); + } + + public String getJsonEncodedStringFromPage() throws MalformedURLException, IOException + { + // Image data only appears on the /read/ page and not on the /view/ one. + URL readUrl = new URL(String.format("http://hentainexus.com/read/%s",getGID(url))); + Document document = Http.url(readUrl).response().parse(); + + for (Element scripts : document.getElementsByTag("script")) { + for (DataNode dataNode : scripts.dataNodes()) { + if (dataNode.getWholeData().contains("initReader")) { + // Extract JSON encoded string from the JavaScript initReader() call. + String data = dataNode.getWholeData().trim().replaceAll("\\r|\\n|\\t",""); + + Pattern p = Pattern.compile(".*?initReader\\(\"(.*?)\",.*?\\).*?"); + Matcher m = p.matcher(data); + if (m.matches()) { + return m.group(1); + } } - Element image = images.first(); - String imgsrc = image.attr("src"); - String prefix = ""; - if (Utils.getConfigBoolean("download.save_order", true)) { - prefix = String.format("%03d_", index); - } - addURLToDownload(new URL(imgsrc), prefix); - } catch (IOException e) { - LOGGER.error("[!] Exception while loading/parsing " + this.url, e); } } + return ""; } -} + + public String decodeJsonString(String jsonEncodedString) + { + /* + The initReader() JavaScript function accepts 2 parameters: a weird string and the window title (we can ignore this). + The weird string is a JSON string with some bytes shifted and swapped around and then encoded in base64. + The following code is a Java adaptation of the initRender() JavaScript function after manual deobfuscation. + */ + + byte[] jsonBytes = Base64.getDecoder().decode(jsonEncodedString); + + ArrayList unknownArray = new ArrayList(); + ArrayList indexesToUse = new ArrayList<>(); + + for (int i = 0x2; unknownArray.size() < 0x10; ++i) { + if (!indexesToUse.contains(i)) { + unknownArray.add(i); + for (int j = i << 0x1; j <= 0x100; j += i) { + if (!indexesToUse.contains(j)) { + indexesToUse.add(j); + } + } + } + } + + byte magicByte = 0x0; + for (int i = 0x0; i < 0x40; i++) { + magicByte = (byte) (signedToUnsigned(magicByte) ^ signedToUnsigned(jsonBytes[i])); + for (int j = 0x0; j < 0x8; j++) { + long unsignedMagicByteTMP = signedToUnsigned(magicByte); + magicByte = (byte) ((unsignedMagicByteTMP & 0x1) == 1 ? unsignedMagicByteTMP >>> 0x1 ^ 0xc : unsignedMagicByteTMP >>> 0x1); + } + } + + magicByte = (byte) (magicByte & 0x7); + ArrayList newArray = new ArrayList(); + + for (int i = 0x0; i < 0x100; i++) { + newArray.add(i); + } + + int newIndex = 0, backup = 0; + for (int i = 0x0; i < 0x100; i++) { + newIndex = (newIndex + newArray.get(i) + (int) signedToUnsigned(jsonBytes[i % 0x40])) % 0x100; + backup = newArray.get(i); + newArray.set(i, newArray.get(newIndex)); + newArray.set(newIndex, backup); + } + + int magicByteTranslated = (int) unknownArray.get(magicByte); + int index1 = 0x0, index2 = 0x0, index3 = 0x0, swap1 = 0x0, xorNumber = 0x0; + String decodedJsonString = ""; + + for (int i = 0x0; i + 0x40 < jsonBytes.length; i++) { + index1 = (index1 + magicByteTranslated) % 0x100; + index2 = (index3 + newArray.get((index2 + newArray.get(index1)) % 0x100)) % 0x100; + index3 = (index3 + index1 + newArray.get(index1)) % 0x100; + swap1 = newArray.get(index1); + newArray.set(index1, newArray.get(index2)); + newArray.set(index2,swap1); + xorNumber = newArray.get((index2 + newArray.get((index1 + newArray.get((xorNumber + index3) % 0x100)) % 0x100)) % 0x100); + decodedJsonString += Character.toString((char) signedToUnsigned((jsonBytes[i + 0x40] ^ xorNumber))); + } + + return decodedJsonString; + } + + + private static long signedToUnsigned(int signed) { + return (byte) signed & 0xFF; + } + +} \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java index cfe540fb..a244276c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java @@ -2,14 +2,43 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; import java.net.URL; +import java.util.ArrayList; +import java.util.List; import com.rarchives.ripme.ripper.rippers.HentaiNexusRipper; +import org.json.JSONObject; +import org.junit.Assert; import org.junit.jupiter.api.Test; public class HentainexusRipperTest extends RippersTest { @Test - public void testHentaiNexusAlbum() throws IOException { - HentaiNexusRipper ripper = new HentaiNexusRipper(new URL("https://hentainexus.com/view/44")); - testRipper(ripper); + public void testHentaiNexusJson() throws IOException { + List testURLs = new ArrayList<>(); + testURLs.add(new URL("https://hentainexus.com/view/9202")); + testURLs.add(new URL("https://hentainexus.com/read/9202")); + testURLs.add(new URL("https://hentainexus.com/view/9202#001")); + testURLs.add(new URL("https://hentainexus.com/read/9202#001")); + + for (URL url : testURLs) { + + HentaiNexusRipper ripper = new HentaiNexusRipper(url); + + boolean testOK = false; + try { + + String jsonEncodedString = ripper.getJsonEncodedStringFromPage(); + String jsonDecodedString = ripper.decodeJsonString(jsonEncodedString); + JSONObject json = new JSONObject(jsonDecodedString); + // Fail test if JSON empty + testOK = !json.isEmpty(); + + } catch (Exception e) { + // Fail test if JSON invalid, not present or other errors + testOK = false; + } + + Assert.assertEquals(true, testOK); + } + } } From 10c3624876694a5d43d72206b8401db9ec44a22e Mon Sep 17 00:00:00 2001 From: PaaaulZ <46759927+PaaaulZ@users.noreply.github.com> Date: Sat, 19 Dec 2020 01:37:36 +0100 Subject: [PATCH 088/512] Fixed #1795 and added new test --- .../com/rarchives/ripme/ripper/rippers/GfycatRipper.java | 6 +++--- .../ripme/tst/ripper/rippers/GfycatRipperTest.java | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java index 37b2d5ae..c542c6dc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java @@ -57,7 +57,7 @@ public class GfycatRipper extends AbstractHTMLRipper { } public boolean isProfile() { - Pattern p = Pattern.compile("^https?://[wm.]*gfycat\\.com/@([a-zA-Z0-9]+).*$"); + Pattern p = Pattern.compile("^https?://[wm.]*gfycat\\.com/@([a-zA-Z0-9\\.\\-\\_]+).*$"); Matcher m = p.matcher(url.toExternalForm()); return m.matches(); } @@ -79,11 +79,11 @@ public class GfycatRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://(thumbs\\.|[wm\\.]*)gfycat\\.com/@?([a-zA-Z0-9]+).*$"); + Pattern p = Pattern.compile("^https?://(?:thumbs\\.|[wm\\.]*)gfycat\\.com/@?([a-zA-Z0-9\\.\\-\\_]+).*$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) - return m.group(2); + return m.group(1); throw new MalformedURLException( "Expected gfycat.com format: " diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java index 019350ad..39c14673 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java @@ -44,4 +44,13 @@ public class GfycatRipperTest extends RippersTest { GfycatRipper ripper = new GfycatRipper(new URL("https://gfycat.com/amp/TemptingExcellentIchthyosaurs")); testRipper(ripper); } + + /** + * Rips a Gfycat profile with special characters in username + * @throws IOException + */ + public void testGfycatSpecialChar() throws IOException { + GfycatRipper ripper = new GfycatRipper(new URL("https://gfycat.com/@rsss.kr")); + testRipper(ripper); + } } From d2ac05f8f515d08966a2412c88e35d6fa74025c4 Mon Sep 17 00:00:00 2001 From: PaaaulZ <46759927+PaaaulZ@users.noreply.github.com> Date: Sat, 2 Jan 2021 03:15:47 +0100 Subject: [PATCH 089/512] Fixed failing test --- .../ripme/tst/ripper/rippers/ScrolllerRipperTest.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java index feef282b..c7bf3d7d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java @@ -10,7 +10,7 @@ import java.util.HashMap; import java.util.Map; public class ScrolllerRipperTest extends RippersTest { - /*@Test + @Test public void testScrolllerGID() throws IOException { Map testURLs = new HashMap<>(); @@ -21,17 +21,16 @@ public class ScrolllerRipperTest extends RippersTest { for (URL url : testURLs.keySet()) { ScrolllerRipper ripper = new ScrolllerRipper(url); ripper.setup(); - System.out.println(testURLs.get(url) + " |=>| " + ripper.getGID(ripper.getURL())); Assertions.assertEquals(testURLs.get(url), ripper.getGID(ripper.getURL())); deleteDir(ripper.getWorkingDir()); } - }*/ + } @Test public void testScrolllerFilterRegex() throws IOException { Map testURLs = new HashMap<>(); - testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp"), ""); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp"), "NOFILTER"); testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=pictures"), "PICTURE"); testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=videos"), "VIDEO"); testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=albums"), "ALBUM"); @@ -44,7 +43,6 @@ public class ScrolllerRipperTest extends RippersTest { for (URL url : testURLs.keySet()) { ScrolllerRipper ripper = new ScrolllerRipper(url); ripper.setup(); - System.out.println(url + " ==> " + testURLs.get(url) + " => " + ripper.convertFilterString(ripper.getParameter(ripper.getURL(),"filter"))); Assertions.assertEquals(testURLs.get(url), ripper.convertFilterString(ripper.getParameter(ripper.getURL(),"filter"))); deleteDir(ripper.getWorkingDir()); } From bfd0b395c7c4e7dad42aa35e4e4e6e626d1dae63 Mon Sep 17 00:00:00 2001 From: PaaaulZ <46759927+PaaaulZ@users.noreply.github.com> Date: Sat, 2 Jan 2021 08:03:00 +0100 Subject: [PATCH 090/512] Added support for sort parameter and WebSocket dependency --- java | 0 pom.xml | 5 + .../ripme/ripper/rippers/ScrolllerRipper.java | 236 ++++++++++++++---- 3 files changed, 188 insertions(+), 53 deletions(-) create mode 100644 java diff --git a/java b/java new file mode 100644 index 00000000..e69de29b diff --git a/pom.xml b/pom.xml index fb1bb42e..5432aa53 100644 --- a/pom.xml +++ b/pom.xml @@ -83,6 +83,11 @@ httpmime 4.3.3 + + org.java-websocket + Java-WebSocket + 1.5.1 + diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java index afadcd1f..a333eac6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java @@ -4,17 +4,21 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; -import java.net.HttpURLConnection; -import java.net.MalformedURLException; -import java.net.URL; +import java.net.*; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.oracle.truffle.js.nodes.access.IteratorStepNode; +import com.oracle.truffle.js.runtime.builtins.JSON; +import org.java_websocket.client.WebSocketClient; + import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; +import org.java_websocket.handshake.ServerHandshake; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -46,7 +50,7 @@ public class ScrolllerRipper extends AbstractJSONRipper { return m.group(1); } throw new MalformedURLException("Expected scrolller.com URL format: " + - "scrolller.com/r/subreddit OR scroller.com/r/subreddit?filter= - got " + url + "instead"); + "scrolller.com/r/subreddit OR scroller.com/r/subreddit?filter= - got " + url + "instead"); } @Override @@ -55,34 +59,56 @@ public class ScrolllerRipper extends AbstractJSONRipper { } - private JSONObject prepareQuery(String iterator, String gid) throws IOException { + private JSONObject prepareQuery(String iterator, String gid, String sortByString) throws IOException, URISyntaxException { // Prepares the JSONObject we need to pass to the GraphQL query. - String queryString = "query SubredditQuery( $url: String! $filter: SubredditPostFilter $iterator: String ) { getSubreddit(url: $url) { children( limit: 50 iterator: $iterator filter: $filter ) { iterator items { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } } } }"; - String filterString = convertFilterString(getParameter(this.url,"filter")); + if (sortByString.equals("")) { + // Sorting not selected + String queryString = "query SubredditQuery( $url: String! $filter: SubredditPostFilter $iterator: String ) { getSubreddit(url: $url) { children( limit: 50 iterator: $iterator filter: $filter ) { iterator items { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } } } }"; + String filterString = convertFilterString(getParameter(this.url,"filter")); - JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid)); + JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid)); - if (iterator != null) { - // Iterator is not present on the first page - variablesObject.put("iterator", iterator); + if (iterator != null) { + // Iterator is not present on the first page + variablesObject.put("iterator", iterator); + } + if (!filterString.equals("NOFILTER")) { + // We could also pass filter="" but not including it if not present is cleaner + variablesObject.put("filter", filterString); + } + + JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", queryString); + + return getPosts(finalQueryObject); } - if (!filterString.equals("NOFILTER")) { - // We could also pass filter="" but not including it if not present is cleaner - variablesObject.put("filter", filterString); + else { + + String queryString = "subscription SubredditSubscription( $url: String! $sortBy: SubredditSortBy $timespan: SubredditTimespan $iterator: String $limit: Int $filter: SubredditPostFilter ) { fetchSubreddit( url: $url sortBy: $sortBy timespan: $timespan iterator: $iterator limit: $limit filter: $filter ) { __typename ... on Subreddit { __typename url title secondaryTitle description createdAt isNsfw subscribers isComplete itemCount videoCount pictureCount albumCount isFollowing } ... on SubredditPost { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } ... on Iterator { iterator } ... on Error { message } } }"; + String filterString = convertFilterString(getParameter(this.url,"filter")); + + JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid)).put("sortBy", sortByString.toUpperCase()); + + if (iterator != null) { + // Iterator is not present on the first page + variablesObject.put("iterator", iterator); + } + if (!filterString.equals("NOFILTER")) { + // We could also pass filter="" but not including it if not present is cleaner + variablesObject.put("filter", filterString); + } + + JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", queryString); + + return getPostsSorted(finalQueryObject); } - - JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", queryString); - - return getPosts(finalQueryObject); - } public String convertFilterString(String filterParameter) { // Converts the ?filter= parameter of the URL to one that can be used in the GraphQL query - // I could basically remove the last "s" and uppercase instead of this switch statement but this looks easier to read. + // I could basically remove the last "s" and call toUpperCase instead of this switch statement but this looks easier to read. switch (filterParameter) { case "pictures": return "PICTURE"; @@ -118,9 +144,7 @@ public class ScrolllerRipper extends AbstractJSONRipper { private JSONObject getPosts(JSONObject data) { // The actual GraphQL query call - // JSoup wants POST data in key=value but I need to write a JSON body so I can't use it... try { - String url = "https://api.scrolller.com/api/v2/graphql"; URL obj = new URL(url); @@ -139,75 +163,181 @@ public class ScrolllerRipper extends AbstractJSONRipper { BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream())); String inputLine; - StringBuffer html = new StringBuffer(); + StringBuffer jsonString = new StringBuffer(); while ((inputLine = in.readLine()) != null) { - html.append(inputLine); + jsonString.append(inputLine); } in.close(); conn.disconnect(); - return new JSONObject(html.toString()); + return new JSONObject(jsonString.toString()); } catch (Exception e) { e.printStackTrace(); } return new JSONObject("{}"); -} + } + + private JSONObject getPostsSorted(JSONObject data) throws MalformedURLException { + + // The actual GraphQL query call (if sort parameter is present) + try { + + ArrayList postsJsonStrings = new ArrayList<>(); + + WebSocketClient wsc = new WebSocketClient(new URI("wss://api.scrolller.com/api/v2/graphql")) { + @Override + public void onOpen(ServerHandshake serverHandshake) { + // As soon as the WebSocket connects send our query + this.send(data.toString()); + } + + @Override + public void onMessage(String s) { + postsJsonStrings.add(s); + if (s.contains("{\"data\":{\"fetchSubreddit\":{\"__typename\":\"Iterator\",\"iterator\":")) { + // Iterator is the last field returned, once we received it we can close the connection. + this.close(); + } + } + + @Override + public void onClose(int i, String s, boolean b) { + } + + @Override + public void onError(Exception e) { + LOGGER.error(String.format("WebSocket error, server reported %s", e.getMessage())); + } + }; + wsc.connect(); + + while (!wsc.isClosed()) { + // Posts list is not over until the connection closes. + } + + JSONObject finalObject = new JSONObject(); + JSONArray posts = new JSONArray(); + + // Iterator is the last object in the post list, let's duplicate it in his own object for clarity. + finalObject.put("iterator", new JSONObject(postsJsonStrings.get(postsJsonStrings.size()-1))); + + for (String postString : postsJsonStrings) { + posts.put(new JSONObject(postString)); + } + finalObject.put("posts", posts); + + return finalObject; + + + } catch (URISyntaxException ue) { + // Nothing to catch, it's an hardcoded URI. + } + + return null; + } @Override protected List getURLsFromJSON(JSONObject json) throws JSONException { - JSONArray itemsList = json.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").getJSONArray("items"); - int bestArea = 0; - String bestUrl = ""; - List list = new ArrayList<>(); + + if (json.has("posts")) { + // If the JSONObject contains the key "posts" it's our custom JSON made after the WebSocket call. + + JSONArray itemsList = json.getJSONArray("posts"); + int bestArea = 0; + String bestUrl = ""; + List list = new ArrayList<>(); - for (Object item : itemsList) { - JSONArray sourcesTMP = ((JSONObject) item).getJSONArray("mediaSources"); - for (Object sourceTMP : sourcesTMP) - { - int widthTMP = ((JSONObject) sourceTMP).getInt("width"); - int heightTMP = ((JSONObject) sourceTMP).getInt("height"); - int areaTMP = widthTMP * heightTMP; + for (Object item : itemsList) { + if (((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { + // Is it really a post? It could be the subreddit description or the iterator (first and last item) + JSONArray sourcesTMP = ((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").getJSONArray("mediaSources"); + for (Object sourceTMP : sourcesTMP) + { + int widthTMP = ((JSONObject) sourceTMP).getInt("width"); + int heightTMP = ((JSONObject) sourceTMP).getInt("height"); + int areaTMP = widthTMP * heightTMP; + + if (areaTMP > bestArea) { + // Better way to determine best image? + bestArea = widthTMP; + bestUrl = ((JSONObject) sourceTMP).getString("url"); + } + } + list.add(bestUrl); + bestUrl = ""; + bestArea = 0; - if (areaTMP > bestArea) { - // Better way to determine best image? - bestArea = widthTMP; - bestUrl = ((JSONObject) sourceTMP).getString("url"); } } - list.add(bestUrl); - bestUrl = ""; - bestArea = 0; + return list; + + } else { + JSONArray itemsList = json.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").getJSONArray("items"); + int bestArea = 0; + String bestUrl = ""; + List list = new ArrayList<>(); + + + for (Object item : itemsList) { + JSONArray sourcesTMP = ((JSONObject) item).getJSONArray("mediaSources"); + for (Object sourceTMP : sourcesTMP) + { + int widthTMP = ((JSONObject) sourceTMP).getInt("width"); + int heightTMP = ((JSONObject) sourceTMP).getInt("height"); + int areaTMP = widthTMP * heightTMP; + + if (areaTMP > bestArea) { + // Better way to determine best image? + bestArea = widthTMP; + bestUrl = ((JSONObject) sourceTMP).getString("url"); + } + } + list.add(bestUrl); + bestUrl = ""; + bestArea = 0; + } + return list; } - return list; } @Override protected JSONObject getFirstPage() throws IOException { - if (getParameter(url,"sort") != null) { - // I need support for the WebSocket protocol to implement sorting. - // A GraphQL query to the API with the "sortBy" variable can't come from a POST request or it will return error 500, it has to come from a WebSocket. - LOGGER.warn("Sorting is not currently implemented and it will be ignored"); + try { + return prepareQuery(null, this.getGID(url), getParameter(url,"sort")); + } catch (URISyntaxException e) { + LOGGER.error(String.format("Error obtaining first page: %s", e.getMessage())); + return null; } - return prepareQuery(null, this.getGID(url)); } @Override public JSONObject getNextPage(JSONObject source) throws IOException { // Every call the the API contains an "iterator" string that we need to pass to the API to get the next page // Checking if iterator is null is not working for some reason, hence why the weird "iterator.toString().equals("null")" - Object iterator = source.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").get("iterator"); + + Object iterator = null; + if (source.has("iterator")) { + // sorted + iterator = source.getJSONObject("iterator").getJSONObject("data").getJSONObject("fetchSubreddit").get("iterator"); + } else { + iterator = source.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").get("iterator"); + } if (!iterator.toString().equals("null")) { - return prepareQuery(iterator.toString(), this.getGID(url)); + try { + return prepareQuery(iterator.toString(), this.getGID(url), getParameter(url,"sort")); + } catch (URISyntaxException e) { + LOGGER.error(String.format("Error changing page: %s", e.getMessage())); + return null; + } } else { return null; } - } } \ No newline at end of file From 72a697eda873829a4b69c348c0045ced66f6b41b Mon Sep 17 00:00:00 2001 From: PaaaulZ <46759927+PaaaulZ@users.noreply.github.com> Date: Sat, 2 Jan 2021 20:36:03 +0100 Subject: [PATCH 091/512] Refactor, removed duplicated code. Fixed bugs --- .../ripme/ripper/rippers/ScrolllerRipper.java | 146 ++++++------------ 1 file changed, 48 insertions(+), 98 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java index a333eac6..7e0c1c46 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java @@ -8,12 +8,9 @@ import java.net.*; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; -import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.oracle.truffle.js.nodes.access.IteratorStepNode; -import com.oracle.truffle.js.runtime.builtins.JSON; import org.java_websocket.client.WebSocketClient; import org.apache.http.NameValuePair; @@ -61,55 +58,31 @@ public class ScrolllerRipper extends AbstractJSONRipper { private JSONObject prepareQuery(String iterator, String gid, String sortByString) throws IOException, URISyntaxException { - // Prepares the JSONObject we need to pass to the GraphQL query. + String QUERY_NOSORT = "query SubredditQuery( $url: String! $filter: SubredditPostFilter $iterator: String ) { getSubreddit(url: $url) { children( limit: 50 iterator: $iterator filter: $filter ) { iterator items { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } } } }"; + String QUERY_SORT = "subscription SubredditSubscription( $url: String! $sortBy: SubredditSortBy $timespan: SubredditTimespan $iterator: String $limit: Int $filter: SubredditPostFilter ) { fetchSubreddit( url: $url sortBy: $sortBy timespan: $timespan iterator: $iterator limit: $limit filter: $filter ) { __typename ... on Subreddit { __typename url title secondaryTitle description createdAt isNsfw subscribers isComplete itemCount videoCount pictureCount albumCount isFollowing } ... on SubredditPost { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } ... on Iterator { iterator } ... on Error { message } } }"; - if (sortByString.equals("")) { - // Sorting not selected - String queryString = "query SubredditQuery( $url: String! $filter: SubredditPostFilter $iterator: String ) { getSubreddit(url: $url) { children( limit: 50 iterator: $iterator filter: $filter ) { iterator items { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } } } }"; - String filterString = convertFilterString(getParameter(this.url,"filter")); + String filterString = convertFilterString(getParameter(this.url,"filter")); - JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid)); + JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid)).put("sortBy", sortByString.toUpperCase()); + JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", sortByString.equals("") ? QUERY_NOSORT : QUERY_SORT); - if (iterator != null) { - // Iterator is not present on the first page - variablesObject.put("iterator", iterator); - } - if (!filterString.equals("NOFILTER")) { - // We could also pass filter="" but not including it if not present is cleaner - variablesObject.put("filter", filterString); - } - - JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", queryString); - - return getPosts(finalQueryObject); + if (iterator != null) { + // Iterator is not present on the first page + variablesObject.put("iterator", iterator); } - else { - - String queryString = "subscription SubredditSubscription( $url: String! $sortBy: SubredditSortBy $timespan: SubredditTimespan $iterator: String $limit: Int $filter: SubredditPostFilter ) { fetchSubreddit( url: $url sortBy: $sortBy timespan: $timespan iterator: $iterator limit: $limit filter: $filter ) { __typename ... on Subreddit { __typename url title secondaryTitle description createdAt isNsfw subscribers isComplete itemCount videoCount pictureCount albumCount isFollowing } ... on SubredditPost { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } ... on Iterator { iterator } ... on Error { message } } }"; - String filterString = convertFilterString(getParameter(this.url,"filter")); - - JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid)).put("sortBy", sortByString.toUpperCase()); - - if (iterator != null) { - // Iterator is not present on the first page - variablesObject.put("iterator", iterator); - } - if (!filterString.equals("NOFILTER")) { - // We could also pass filter="" but not including it if not present is cleaner - variablesObject.put("filter", filterString); - } - - JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", queryString); - - return getPostsSorted(finalQueryObject); + if (!filterString.equals("NOFILTER")) { + variablesObject.put("filter", filterString); } + + return sortByString.equals("") ? getPosts(finalQueryObject) : getPostsSorted(finalQueryObject); + } public String convertFilterString(String filterParameter) { // Converts the ?filter= parameter of the URL to one that can be used in the GraphQL query // I could basically remove the last "s" and call toUpperCase instead of this switch statement but this looks easier to read. - switch (filterParameter) { + switch (filterParameter.toLowerCase()) { case "pictures": return "PICTURE"; case "videos": @@ -134,7 +107,7 @@ public class ScrolllerRipper extends AbstractJSONRipper { // 1) arg.getName() => https://scrolller.com/r/CatsStandingUp?filter // 2) arg.getName() => sort - if (arg.getName().replace(toReplace,"").equals((parameter))) { + if (arg.getName().replace(toReplace,"").toLowerCase().equals((parameter))) { return arg.getValue(); } } @@ -198,8 +171,7 @@ public class ScrolllerRipper extends AbstractJSONRipper { @Override public void onMessage(String s) { postsJsonStrings.add(s); - if (s.contains("{\"data\":{\"fetchSubreddit\":{\"__typename\":\"Iterator\",\"iterator\":")) { - // Iterator is the last field returned, once we received it we can close the connection. + if (new JSONObject(s).getJSONObject("data").getJSONObject("fetchSubreddit").has("iterator")) { this.close(); } } @@ -230,6 +202,11 @@ public class ScrolllerRipper extends AbstractJSONRipper { } finalObject.put("posts", posts); + if (finalObject.getJSONArray("posts").length() == 1 && !finalObject.getJSONArray("posts").getJSONObject(0).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { + // Only iterator, no posts. + return null; + } + return finalObject; @@ -244,66 +221,38 @@ public class ScrolllerRipper extends AbstractJSONRipper { @Override protected List getURLsFromJSON(JSONObject json) throws JSONException { - if (json.has("posts")) { - // If the JSONObject contains the key "posts" it's our custom JSON made after the WebSocket call. + boolean sortRequested = json.has("posts"); - JSONArray itemsList = json.getJSONArray("posts"); - int bestArea = 0; - String bestUrl = ""; - List list = new ArrayList<>(); + int bestArea = 0; + String bestUrl = ""; + List list = new ArrayList<>(); + JSONArray itemsList = sortRequested ? json.getJSONArray("posts") : json.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").getJSONArray("items"); - for (Object item : itemsList) { - if (((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { - // Is it really a post? It could be the subreddit description or the iterator (first and last item) - JSONArray sourcesTMP = ((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").getJSONArray("mediaSources"); - for (Object sourceTMP : sourcesTMP) - { - int widthTMP = ((JSONObject) sourceTMP).getInt("width"); - int heightTMP = ((JSONObject) sourceTMP).getInt("height"); - int areaTMP = widthTMP * heightTMP; + for (Object item : itemsList) { - if (areaTMP > bestArea) { - // Better way to determine best image? - bestArea = widthTMP; - bestUrl = ((JSONObject) sourceTMP).getString("url"); - } - } - list.add(bestUrl); - bestUrl = ""; - bestArea = 0; + if (sortRequested && !((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { + continue; + } + JSONArray sourcesTMP = sortRequested ? ((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").getJSONArray("mediaSources") : ((JSONObject) item).getJSONArray("mediaSources"); + for (Object sourceTMP : sourcesTMP) + { + int widthTMP = ((JSONObject) sourceTMP).getInt("width"); + int heightTMP = ((JSONObject) sourceTMP).getInt("height"); + int areaTMP = widthTMP * heightTMP; + + if (areaTMP > bestArea) { + bestArea = widthTMP; + bestUrl = ((JSONObject) sourceTMP).getString("url"); } } - return list; - - } else { - JSONArray itemsList = json.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").getJSONArray("items"); - int bestArea = 0; - String bestUrl = ""; - List list = new ArrayList<>(); - - - for (Object item : itemsList) { - JSONArray sourcesTMP = ((JSONObject) item).getJSONArray("mediaSources"); - for (Object sourceTMP : sourcesTMP) - { - int widthTMP = ((JSONObject) sourceTMP).getInt("width"); - int heightTMP = ((JSONObject) sourceTMP).getInt("height"); - int areaTMP = widthTMP * heightTMP; - - if (areaTMP > bestArea) { - // Better way to determine best image? - bestArea = widthTMP; - bestUrl = ((JSONObject) sourceTMP).getString("url"); - } - } - list.add(bestUrl); - bestUrl = ""; - bestArea = 0; - } - return list; + list.add(bestUrl); + bestUrl = ""; + bestArea = 0; } + + return list; } @Override @@ -323,12 +272,14 @@ public class ScrolllerRipper extends AbstractJSONRipper { Object iterator = null; if (source.has("iterator")) { - // sorted + // Sort requested, custom JSON. iterator = source.getJSONObject("iterator").getJSONObject("data").getJSONObject("fetchSubreddit").get("iterator"); } else { iterator = source.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").get("iterator"); } + if (!iterator.toString().equals("null")) { + // Need to change page. try { return prepareQuery(iterator.toString(), this.getGID(url), getParameter(url,"sort")); } catch (URISyntaxException e) { @@ -339,5 +290,4 @@ public class ScrolllerRipper extends AbstractJSONRipper { return null; } } - } \ No newline at end of file From e967394eb7faa848f3414c4e5ec7b0c3186f758d Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 26 Nov 2020 17:42:00 +0100 Subject: [PATCH 092/512] fix xhamster testGetNextPage --- .../com/rarchives/ripme/ripper/rippers/XhamsterRipper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 7ae570f3..7d495fec 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -139,8 +139,8 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException { - if (doc.select("a[data-page=next]").first() != null) { - String nextPageUrl = doc.select("a[data-page=next]").first().attr("href"); + if (doc.select("a[rel=next]").first() != null) { + String nextPageUrl = doc.select("a[rel=next]").first().attr("href"); if (nextPageUrl.startsWith("http")) { nextPageUrl = nextPageUrl.replaceAll("https?://\\w?\\w?\\.?xhamster\\.", "https://m.xhamster."); nextPageUrl = nextPageUrl.replaceAll("https?://xhamster2\\.", "https://m.xhamster2."); From 2cb6ae714f31d9e4b65648f7fea96c9540a6e4e5 Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 26 Nov 2020 18:11:22 +0100 Subject: [PATCH 093/512] mark ImagefapRipperTest and YoupornRipperTest flaky --- .../rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java | 1 + .../rarchives/ripme/tst/ripper/rippers/YoupornRipperTest.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java index 0ed0add2..19061e34 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java @@ -30,6 +30,7 @@ public class ImagefapRipperTest extends RippersTest { } } @Test + @Tag("flaky") public void testImagefapGetAlbumTitle() throws IOException { URL url = new URL("https://www.imagefap.com/gallery.php?gid=7789753"); ImagefapRipper ripper = new ImagefapRipper(url); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YoupornRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YoupornRipperTest.java index c8640cad..bce22d62 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YoupornRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YoupornRipperTest.java @@ -1,6 +1,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.YoupornRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -10,6 +11,7 @@ import java.util.List; public class YoupornRipperTest extends RippersTest { @Test + @Tag("flaky") public void testYoupornRipper() throws IOException { List contentURLs = new ArrayList<>(); contentURLs.add(new URL("http://www.youporn.com/watch/7669155/mrs-li-amateur-69-orgasm/?from=categ")); From f7b85aa06a166b6a197139a67a5f2848b1cd1c8e Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 26 Nov 2020 20:06:59 +0100 Subject: [PATCH 094/512] xhamster5.desi is a valid domain --- .../ripme/ripper/rippers/XhamsterRipper.java | 29 +++++++++---------- .../ripper/rippers/XhamsterRipperTest.java | 6 ++-- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 7d495fec..7ade1e55 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -48,8 +48,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { return url; } String URLToReturn = url.toExternalForm(); - URLToReturn = URLToReturn.replaceAll("https?://\\w?\\w?\\.?xhamster\\.", "https://m.xhamster."); - URLToReturn = URLToReturn.replaceAll("https?://xhamster2\\.", "https://m.xhamster2."); + URLToReturn = URLToReturn.replaceAll("https?://\\w?\\w?\\.?xhamster([^<]*)\\.", "https://m.xhamster$1."); URL san_url = new URL(URLToReturn); LOGGER.info("sanitized URL is " + san_url.toExternalForm()); return san_url; @@ -57,20 +56,20 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.com/photos/gallery/.*?(\\d+)$"); + Pattern p = Pattern.compile("^https?://([\\w\\w]*\\.)?xhamster([^<]*)\\.(com|one|desi)/photos/gallery/.*?(\\d+)$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { - return m.group(1); + return m.group(4); } - p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.com/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); + p = Pattern.compile("^https?://[\\w\\w.]*xhamster([^<]*)\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return "user_" + m.group(1); } - p = Pattern.compile("^https?://.*xhamster2?\\.com/(movies|videos)/(.*)$"); + p = Pattern.compile("^https?://.*xhamster([^<]*)\\.(com|one|desi)/(movies|videos)/(.*$)"); m = p.matcher(url.toExternalForm()); if (m.matches()) { - return m.group(2); + return m.group(4); } throw new MalformedURLException( @@ -97,7 +96,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public boolean pageContainsAlbums(URL url) { - Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster\\.com/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); + Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster([^<]*)\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); Matcher m = p.matcher(url.toExternalForm()); LOGGER.info("Checking if page has albums"); LOGGER.info(m.matches()); @@ -113,17 +112,17 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public boolean canRip(URL url) { - Pattern p = Pattern.compile("^https?://([\\w\\w]*\\.)?xhamster2?\\.(com|one|desi)/photos/gallery/.*?(\\d+)$"); + Pattern p = Pattern.compile("^https?://([\\w\\w]*\\.)?xhamster([^<]*)\\.(com|one|desi)/photos/gallery/.*?(\\d+)$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return true; } - p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); + p = Pattern.compile("^https?://[\\w\\w.]*xhamster([^<]*)\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return true; } - p = Pattern.compile("^https?://.*xhamster2?\\.(com|one|desi)/(movies|videos)/.*$"); + p = Pattern.compile("^https?://.*xhamster([^<]*)\\.(com|one|desi)/(movies|videos)/(.*$)"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return true; @@ -132,7 +131,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { } private boolean isVideoUrl(URL url) { - Pattern p = Pattern.compile("^https?://.*xhamster2?\\.(com|one|desi)/(movies|videos)/.*$"); + Pattern p = Pattern.compile("^https?://.*xhamster([^<]*)\\.(com|one|desi)/(movies|videos)/(.*$)"); Matcher m = p.matcher(url.toExternalForm()); return m.matches(); } @@ -142,8 +141,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { if (doc.select("a[rel=next]").first() != null) { String nextPageUrl = doc.select("a[rel=next]").first().attr("href"); if (nextPageUrl.startsWith("http")) { - nextPageUrl = nextPageUrl.replaceAll("https?://\\w?\\w?\\.?xhamster\\.", "https://m.xhamster."); - nextPageUrl = nextPageUrl.replaceAll("https?://xhamster2\\.", "https://m.xhamster2."); + nextPageUrl = nextPageUrl.replaceAll("https?://\\w?\\w?\\.?xhamster([^<]*)\\.", "https://m.xhamster$1."); return Http.url(nextPageUrl).get(); } } @@ -165,8 +163,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { try { // This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to // the page chamster.com but displays the mobile site from m.xhamster.com - pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster\\.", "://m.xhamster."); - pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster2\\.", "://m.xhamster."); + pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster([^<]*)\\.", "://m.xhamster$1."); String image = Http.url(new URL(pageWithImageUrl)).get().select("a > img#photoCurr").attr("src"); downloadFile(image); } catch (IOException e) { diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java index a5f1beaa..e9475a1e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java @@ -35,7 +35,7 @@ public class XhamsterRipperTest extends RippersTest { @Test @Tag("flaky") public void testXhamsterAlbumDesiDomain() throws IOException { - XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.desi/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664")); + XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster5.desi/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664")); testRipper(ripper); } @Test @@ -49,9 +49,9 @@ public class XhamsterRipperTest extends RippersTest { XhamsterRipper ripper = new XhamsterRipper(new URL("https://pt.xhamster.com/photos/gallery/silvana-7105696")); testRipper(ripper); } - + @Test public void testGetGID() throws IOException { - URL url = new URL("https://xhamster.com/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664"); + URL url = new URL("https://xhamster5.desi/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664"); XhamsterRipper ripper = new XhamsterRipper(url); Assertions.assertEquals("7254664", ripper.getGID(url)); } From 669ef3a9f4d59f275f1b9310959b90b7440c7c92 Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 26 Nov 2020 20:22:15 +0100 Subject: [PATCH 095/512] redditrippertest flaky --- .../com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java index e99df56c..f4dbe327 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java @@ -61,6 +61,7 @@ public class RedditRipperTest extends RippersTest { } @Test + @Tag("flaky") public void testRedditGallery() throws IOException{ RedditRipper ripper = new RedditRipper( new URL("https://www.reddit.com/gallery/hrrh23")); From 6b87b9c180c6285be78583d0449ae99e1ed89ee4 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 25 Dec 2020 16:47:01 +0100 Subject: [PATCH 096/512] replace nonexisting url with existing in test nextpage --- .../rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java index d0e8dd6a..354b4e62 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java @@ -29,7 +29,7 @@ public class PornhubRipperTest extends RippersTest { @Test public void testGetNextPage() throws IOException { - String baseURL = "https://www.pornhub.com/album/43902391"; + String baseURL = "https://www.pornhub.com/album/30687901"; PornhubRipper ripper = new PornhubRipper(new URL(baseURL)); Document page = Http.url(baseURL).get(); int numPagesRemaining = 1; From d9703782d264ae6e904ba4d00d27d80ac66bf234 Mon Sep 17 00:00:00 2001 From: PaaaulZ <46759927+PaaaulZ@users.noreply.github.com> Date: Fri, 18 Dec 2020 22:17:41 +0100 Subject: [PATCH 097/512] Fixed ripper for HentaiNexus --- .../ripper/rippers/HentaiNexusRipper.java | 216 +++++++++++------- .../ripper/rippers/HentainexusRipperTest.java | 35 ++- 2 files changed, 167 insertions(+), 84 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java index 56ce0d2f..ca709418 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java @@ -4,27 +4,22 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.rarchives.ripme.utils.Http; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import com.rarchives.ripme.ripper.AbstractJSONRipper; +import org.jsoup.nodes.DataNode; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; -import com.rarchives.ripme.utils.Http; -import com.rarchives.ripme.utils.Utils; - -public class HentaiNexusRipper extends AbstractHTMLRipper { - - private Document firstPage; - private DownloadThreadPool hentainexusThreadPool = new DownloadThreadPool("hentainexus"); - @Override - public DownloadThreadPool getThreadPool() { - return hentainexusThreadPool; - } +public class HentaiNexusRipper extends AbstractJSONRipper { public HentaiNexusRipper(URL url) throws IOException { super(url); @@ -34,7 +29,6 @@ public class HentaiNexusRipper extends AbstractHTMLRipper { public String getHost() { return "hentainexus"; } - @Override public String getDomain() { return "hentainexus.com"; @@ -42,88 +36,148 @@ public class HentaiNexusRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("https?://hentainexus\\.com/view/([a-zA-Z0-9_\\-%]*)/?$"); + /* + Valid URLs are /view/id, /read/id and those 2 with #pagenumber + https://hentainexus.com/view/9202 + https://hentainexus.com/read/9202 + https://hentainexus.com/view/9202#001 + https://hentainexus.com/read/9202#001 + */ + + Pattern p = Pattern.compile("^https?://hentainexus\\.com/(?:view|read)/([0-9]+)(?:\\#[0-9]+)*$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); } throw new MalformedURLException("Expected hentainexus.com URL format: " + - "hentainexus.com/view/NUMBER - got " + url + " instead"); - } - - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - if (firstPage == null) { - firstPage = Http.url(url).get(); - } - return firstPage; - } - - @Override - public List getURLsFromPage(Document doc) { - List imageURLs = new ArrayList<>(); - Elements thumbs = doc.select("div.is-multiline > div.column > a"); - for (Element el : thumbs) { - imageURLs.add("https://" + getDomain() + el.attr("href")); - } - return imageURLs; - } - - @Override - public String getAlbumTitle(URL url) throws MalformedURLException { - try { - Document gallery = Http.url(url).get(); - return getHost() + "_" + gallery.select("h1.title").text(); - } catch (IOException e) { - LOGGER.info("Falling back"); - } - - return super.getAlbumTitle(url); + "hentainexus.com/view/id OR hentainexus.com/read/id - got " + url + "instead"); } @Override public void downloadURL(URL url, int index) { - HentaiNexusImageThread t = new HentaiNexusImageThread(url, index); - hentainexusThreadPool.addThread(t); + addURLToDownload(url, getPrefix(index)); } - /** - * Helper class to find and download images found on "image" pages - */ - private class HentaiNexusImageThread extends Thread { - private URL url; - private int index; - HentaiNexusImageThread(URL url, int index) { - super(); - this.url = url; - this.index = index; + @Override + protected List getURLsFromJSON(JSONObject json) throws JSONException { + + List urlList = new ArrayList<>(); + + JSONArray imagesList = json.getJSONArray("f"); + String host = json.getString("b"); + String folder = json.getString("r"); + String id = json.getString("i"); + + for (Object singleImage : imagesList) { + String hashTMP = ((JSONObject) singleImage).getString("h"); + String fileNameTMP = ((JSONObject) singleImage).getString("p"); + String imageUrlTMP = String.format("%s%s%s/%s/%s",host,folder,hashTMP,id,fileNameTMP); + urlList.add(imageUrlTMP); } - @Override - public void run() { - fetchImage(); - } + return urlList; + } - private void fetchImage() { - try { - Document doc = Http.url(url).retries(3).get(); - Elements images = doc.select("figure.image > img"); - if (images.isEmpty()) { - LOGGER.warn("Image not found at " + this.url); - return; + @Override + protected JSONObject getFirstPage() throws IOException { + String jsonEncodedString = getJsonEncodedStringFromPage(); + String jsonDecodedString = decodeJsonString(jsonEncodedString); + return new JSONObject(jsonDecodedString); + } + + public String getJsonEncodedStringFromPage() throws MalformedURLException, IOException + { + // Image data only appears on the /read/ page and not on the /view/ one. + URL readUrl = new URL(String.format("http://hentainexus.com/read/%s",getGID(url))); + Document document = Http.url(readUrl).response().parse(); + + for (Element scripts : document.getElementsByTag("script")) { + for (DataNode dataNode : scripts.dataNodes()) { + if (dataNode.getWholeData().contains("initReader")) { + // Extract JSON encoded string from the JavaScript initReader() call. + String data = dataNode.getWholeData().trim().replaceAll("\\r|\\n|\\t",""); + + Pattern p = Pattern.compile(".*?initReader\\(\"(.*?)\",.*?\\).*?"); + Matcher m = p.matcher(data); + if (m.matches()) { + return m.group(1); + } } - Element image = images.first(); - String imgsrc = image.attr("src"); - String prefix = ""; - if (Utils.getConfigBoolean("download.save_order", true)) { - prefix = String.format("%03d_", index); - } - addURLToDownload(new URL(imgsrc), prefix); - } catch (IOException e) { - LOGGER.error("[!] Exception while loading/parsing " + this.url, e); } } + return ""; } -} + + public String decodeJsonString(String jsonEncodedString) + { + /* + The initReader() JavaScript function accepts 2 parameters: a weird string and the window title (we can ignore this). + The weird string is a JSON string with some bytes shifted and swapped around and then encoded in base64. + The following code is a Java adaptation of the initRender() JavaScript function after manual deobfuscation. + */ + + byte[] jsonBytes = Base64.getDecoder().decode(jsonEncodedString); + + ArrayList unknownArray = new ArrayList(); + ArrayList indexesToUse = new ArrayList<>(); + + for (int i = 0x2; unknownArray.size() < 0x10; ++i) { + if (!indexesToUse.contains(i)) { + unknownArray.add(i); + for (int j = i << 0x1; j <= 0x100; j += i) { + if (!indexesToUse.contains(j)) { + indexesToUse.add(j); + } + } + } + } + + byte magicByte = 0x0; + for (int i = 0x0; i < 0x40; i++) { + magicByte = (byte) (signedToUnsigned(magicByte) ^ signedToUnsigned(jsonBytes[i])); + for (int j = 0x0; j < 0x8; j++) { + long unsignedMagicByteTMP = signedToUnsigned(magicByte); + magicByte = (byte) ((unsignedMagicByteTMP & 0x1) == 1 ? unsignedMagicByteTMP >>> 0x1 ^ 0xc : unsignedMagicByteTMP >>> 0x1); + } + } + + magicByte = (byte) (magicByte & 0x7); + ArrayList newArray = new ArrayList(); + + for (int i = 0x0; i < 0x100; i++) { + newArray.add(i); + } + + int newIndex = 0, backup = 0; + for (int i = 0x0; i < 0x100; i++) { + newIndex = (newIndex + newArray.get(i) + (int) signedToUnsigned(jsonBytes[i % 0x40])) % 0x100; + backup = newArray.get(i); + newArray.set(i, newArray.get(newIndex)); + newArray.set(newIndex, backup); + } + + int magicByteTranslated = (int) unknownArray.get(magicByte); + int index1 = 0x0, index2 = 0x0, index3 = 0x0, swap1 = 0x0, xorNumber = 0x0; + String decodedJsonString = ""; + + for (int i = 0x0; i + 0x40 < jsonBytes.length; i++) { + index1 = (index1 + magicByteTranslated) % 0x100; + index2 = (index3 + newArray.get((index2 + newArray.get(index1)) % 0x100)) % 0x100; + index3 = (index3 + index1 + newArray.get(index1)) % 0x100; + swap1 = newArray.get(index1); + newArray.set(index1, newArray.get(index2)); + newArray.set(index2,swap1); + xorNumber = newArray.get((index2 + newArray.get((index1 + newArray.get((xorNumber + index3) % 0x100)) % 0x100)) % 0x100); + decodedJsonString += Character.toString((char) signedToUnsigned((jsonBytes[i + 0x40] ^ xorNumber))); + } + + return decodedJsonString; + } + + + private static long signedToUnsigned(int signed) { + return (byte) signed & 0xFF; + } + +} \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java index cfe540fb..a244276c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java @@ -2,14 +2,43 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; import java.net.URL; +import java.util.ArrayList; +import java.util.List; import com.rarchives.ripme.ripper.rippers.HentaiNexusRipper; +import org.json.JSONObject; +import org.junit.Assert; import org.junit.jupiter.api.Test; public class HentainexusRipperTest extends RippersTest { @Test - public void testHentaiNexusAlbum() throws IOException { - HentaiNexusRipper ripper = new HentaiNexusRipper(new URL("https://hentainexus.com/view/44")); - testRipper(ripper); + public void testHentaiNexusJson() throws IOException { + List testURLs = new ArrayList<>(); + testURLs.add(new URL("https://hentainexus.com/view/9202")); + testURLs.add(new URL("https://hentainexus.com/read/9202")); + testURLs.add(new URL("https://hentainexus.com/view/9202#001")); + testURLs.add(new URL("https://hentainexus.com/read/9202#001")); + + for (URL url : testURLs) { + + HentaiNexusRipper ripper = new HentaiNexusRipper(url); + + boolean testOK = false; + try { + + String jsonEncodedString = ripper.getJsonEncodedStringFromPage(); + String jsonDecodedString = ripper.decodeJsonString(jsonEncodedString); + JSONObject json = new JSONObject(jsonDecodedString); + // Fail test if JSON empty + testOK = !json.isEmpty(); + + } catch (Exception e) { + // Fail test if JSON invalid, not present or other errors + testOK = false; + } + + Assert.assertEquals(true, testOK); + } + } } From 8d7df4f42f7c6a47683c743d7c305e6653a35f3a Mon Sep 17 00:00:00 2001 From: PaaaulZ <46759927+PaaaulZ@users.noreply.github.com> Date: Sat, 19 Dec 2020 01:37:36 +0100 Subject: [PATCH 098/512] Fixed #1795 and added new test --- .../com/rarchives/ripme/ripper/rippers/GfycatRipper.java | 6 +++--- .../ripme/tst/ripper/rippers/GfycatRipperTest.java | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java index 37b2d5ae..c542c6dc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java @@ -57,7 +57,7 @@ public class GfycatRipper extends AbstractHTMLRipper { } public boolean isProfile() { - Pattern p = Pattern.compile("^https?://[wm.]*gfycat\\.com/@([a-zA-Z0-9]+).*$"); + Pattern p = Pattern.compile("^https?://[wm.]*gfycat\\.com/@([a-zA-Z0-9\\.\\-\\_]+).*$"); Matcher m = p.matcher(url.toExternalForm()); return m.matches(); } @@ -79,11 +79,11 @@ public class GfycatRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://(thumbs\\.|[wm\\.]*)gfycat\\.com/@?([a-zA-Z0-9]+).*$"); + Pattern p = Pattern.compile("^https?://(?:thumbs\\.|[wm\\.]*)gfycat\\.com/@?([a-zA-Z0-9\\.\\-\\_]+).*$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) - return m.group(2); + return m.group(1); throw new MalformedURLException( "Expected gfycat.com format: " diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java index 019350ad..39c14673 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java @@ -44,4 +44,13 @@ public class GfycatRipperTest extends RippersTest { GfycatRipper ripper = new GfycatRipper(new URL("https://gfycat.com/amp/TemptingExcellentIchthyosaurs")); testRipper(ripper); } + + /** + * Rips a Gfycat profile with special characters in username + * @throws IOException + */ + public void testGfycatSpecialChar() throws IOException { + GfycatRipper ripper = new GfycatRipper(new URL("https://gfycat.com/@rsss.kr")); + testRipper(ripper); + } } From cf8f7f7708dfc96ae64a632f6d8d3e2e61167ae7 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 21 Nov 2020 15:09:38 +0100 Subject: [PATCH 099/512] mvn -B talks less when downloading dependencies in check builds --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index bb44b0c8..bb2c54d8 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -18,4 +18,4 @@ jobs: with: java-version: ${{ matrix.java }} - name: Build with Maven - run: mvn package --file pom.xml + run: mvn -B package --file pom.xml From 1751206d7fd2fba059a24e6d70c471d6bf78dd26 Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 26 Nov 2020 22:37:06 +0100 Subject: [PATCH 100/512] one build is sufficient to test newest java check builds are flaky, and doing them more often in a short time frame makes them even more flaky. --- .github/workflows/maven.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index bb2c54d8..bff98872 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -9,7 +9,10 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - java: [1.8, 1.14] + java: [1.8] + include: # test newest java on one os only + - os: ubuntu-latest + java: 1.14 steps: - uses: actions/checkout@v1 From 493b5f9be5618b9fbba5ee5c98fe5c1c11d05b93 Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 27 Oct 2020 05:21:12 +0100 Subject: [PATCH 101/512] upload resulting jar as actions asset, in case of commit to main extend matrix build to produce a fat jar so an executable jar is uploaded. unfortunately it zips the files. addresses #1766, #1820. --- .github/workflows/maven.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index bff98872..35473f78 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -10,9 +10,12 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] java: [1.8] - include: # test newest java on one os only + include: # test newest java on one os only, upload from ubuntu java8 - os: ubuntu-latest java: 1.14 + - os: ubuntu-latest + java: 1.8 + upload: true steps: - uses: actions/checkout@v1 @@ -21,4 +24,12 @@ jobs: with: java-version: ${{ matrix.java }} - name: Build with Maven - run: mvn -B package --file pom.xml + run: mvn -B package assembly:single --file pom.xml + - name: upload jar as asset + if: matrix.upload && github.ref == 'main' + uses: actions/upload-artifact@v2 + with: + name: zipped-ripme-jar + path: target/*dependencies.jar + +# vim:set ts=2 sw=2 et: From 229a96cbab2717074d912dd8ae128b9c5fcbeec3 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 16 Jan 2021 17:45:42 +0100 Subject: [PATCH 102/512] WordpressComicRipperTest flaky --- .../ripme/tst/ripper/rippers/WordpressComicRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java index 6a647286..e8f21726 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java @@ -51,6 +51,7 @@ public class WordpressComicRipperTest extends RippersTest { testRipper(ripper); } @Test + @Tag("flaky") public void test_konradokonski_1() throws IOException { WordpressComicRipper ripper = new WordpressComicRipper( new URL("http://www.konradokonski.com/sawdust/comic/get-up/")); @@ -58,6 +59,7 @@ public class WordpressComicRipperTest extends RippersTest { } @Test + @Tag("flaky") public void test_konradokonski_2() throws IOException { WordpressComicRipper ripper = new WordpressComicRipper( new URL("http://www.konradokonski.com/wiory/comic/08182008/")); From 76aa90a31469f4a50128fa38963bc85b3c855f24 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 26 Jul 2020 08:21:51 +0200 Subject: [PATCH 103/512] ignore gradle bild dirs --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index e7813bc7..fb5ed210 100644 --- a/.gitignore +++ b/.gitignore @@ -80,6 +80,12 @@ buildNumber.properties # Avoid ignoring Maven wrapper jar file (.jar files are usually ignored) !/.mvn/wrapper/maven-wrapper.jar +### gradle ### +/.gradle +/build +# Avoid ignoring gradle wrapper jar file (.jar files are usually ignored) +!/gradle/wrapper/gradle-wrapper.jar + ### Windows ### # Windows thumbnail cache files Thumbs.db From 2c5421d6de064b91b24bbaaf41f3567d19d06e98 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 16 Jan 2021 18:18:15 +0100 Subject: [PATCH 104/512] java-15 in test, jacoco-0.86 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fb1bb42e..8499f9f1 100644 --- a/pom.xml +++ b/pom.xml @@ -132,7 +132,7 @@ org.jacoco jacoco-maven-plugin - 0.8.5 + 0.8.6 prepare-agent From f4ab50127064ff9c6253b1df28c118bf314ae7aa Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 16 Jan 2021 19:34:19 +0100 Subject: [PATCH 105/512] test upload without condition --- .github/workflows/maven.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 35473f78..19b99691 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -14,7 +14,6 @@ jobs: - os: ubuntu-latest java: 1.14 - os: ubuntu-latest - java: 1.8 upload: true steps: @@ -26,7 +25,7 @@ jobs: - name: Build with Maven run: mvn -B package assembly:single --file pom.xml - name: upload jar as asset - if: matrix.upload && github.ref == 'main' + if: matrix.upload uses: actions/upload-artifact@v2 with: name: zipped-ripme-jar From f4b49f0da841851cfa8350d32c5f671cbe137cee Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 23 Jan 2021 12:24:01 +0100 Subject: [PATCH 106/512] redgifsearch, test or ripper broken --- .../rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java index 9789417d..8b45594d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java @@ -46,6 +46,7 @@ public class RedgifsRipperTest extends RippersTest { * @throws IOException */ @Test + @Disabled("test or ripper broken") public void testRedgifsSearch() throws IOException { RedgifsRipper ripper = new RedgifsRipper(new URL("https://redgifs.com/gifs/browse/little-caprice")); Document doc = ripper.getFirstPage(); From 3912eba2479d5d4f61c6354ddd8bed1126d88102 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 23 Jan 2021 12:24:01 +0100 Subject: [PATCH 107/512] redgifsearch, test or ripper broken --- .../rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java index 9789417d..8b45594d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java @@ -46,6 +46,7 @@ public class RedgifsRipperTest extends RippersTest { * @throws IOException */ @Test + @Disabled("test or ripper broken") public void testRedgifsSearch() throws IOException { RedgifsRipper ripper = new RedgifsRipper(new URL("https://redgifs.com/gifs/browse/little-caprice")); Document doc = ripper.getFirstPage(); From 17136bd00176a8d16e6bd4a8865291709c673b40 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 23 Jan 2021 12:39:53 +0100 Subject: [PATCH 108/512] hentai2 test flaky --- .../ripme/tst/ripper/rippers/Hentai2readRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java index f448f0de..c6e2d3de 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.Hentai2readRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class Hentai2readRipperTest extends RippersTest { @Test + @Tag("flaky") public void testHentai2readAlbum() throws IOException { Hentai2readRipper ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/1/")); testRipper(ripper); From 254f9eafbd7e4047f118a2a8c4231a1405927770 Mon Sep 17 00:00:00 2001 From: PaaaulZ <46759927+PaaaulZ@users.noreply.github.com> Date: Thu, 17 Dec 2020 01:45:41 +0100 Subject: [PATCH 109/512] Added support for scrolller.com --- java | 0 pom.xml | 5 + .../ripme/ripper/rippers/ScrolllerRipper.java | 293 ++++++++++++++++++ .../ripper/rippers/ScrolllerRipperTest.java | 53 ++++ 4 files changed, 351 insertions(+) create mode 100644 java create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java diff --git a/java b/java new file mode 100644 index 00000000..e69de29b diff --git a/pom.xml b/pom.xml index 8499f9f1..ccfa46a9 100644 --- a/pom.xml +++ b/pom.xml @@ -83,6 +83,11 @@ httpmime 4.3.3 + + org.java-websocket + Java-WebSocket + 1.5.1 + diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java new file mode 100644 index 00000000..7e0c1c46 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java @@ -0,0 +1,293 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.net.*; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.java_websocket.client.WebSocketClient; + +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; +import org.java_websocket.handshake.ServerHandshake; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import com.rarchives.ripme.ripper.AbstractJSONRipper; + +public class ScrolllerRipper extends AbstractJSONRipper { + + public ScrolllerRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "scrolller"; + } + @Override + public String getDomain() { + return "scrolller.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + // Typical URL is: https://scrolller.com/r/subreddit + // Parameters like "filter" and "sort" can be passed (ex: https://scrolller.com/r/subreddit?filter=xxx&sort=yyyy) + Pattern p = Pattern.compile("^https?://scrolller\\.com/r/([a-zA-Z0-9]+).*?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected scrolller.com URL format: " + + "scrolller.com/r/subreddit OR scroller.com/r/subreddit?filter= - got " + url + "instead"); + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + + + private JSONObject prepareQuery(String iterator, String gid, String sortByString) throws IOException, URISyntaxException { + + String QUERY_NOSORT = "query SubredditQuery( $url: String! $filter: SubredditPostFilter $iterator: String ) { getSubreddit(url: $url) { children( limit: 50 iterator: $iterator filter: $filter ) { iterator items { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } } } }"; + String QUERY_SORT = "subscription SubredditSubscription( $url: String! $sortBy: SubredditSortBy $timespan: SubredditTimespan $iterator: String $limit: Int $filter: SubredditPostFilter ) { fetchSubreddit( url: $url sortBy: $sortBy timespan: $timespan iterator: $iterator limit: $limit filter: $filter ) { __typename ... on Subreddit { __typename url title secondaryTitle description createdAt isNsfw subscribers isComplete itemCount videoCount pictureCount albumCount isFollowing } ... on SubredditPost { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } ... on Iterator { iterator } ... on Error { message } } }"; + + String filterString = convertFilterString(getParameter(this.url,"filter")); + + JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid)).put("sortBy", sortByString.toUpperCase()); + JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", sortByString.equals("") ? QUERY_NOSORT : QUERY_SORT); + + if (iterator != null) { + // Iterator is not present on the first page + variablesObject.put("iterator", iterator); + } + if (!filterString.equals("NOFILTER")) { + variablesObject.put("filter", filterString); + } + + return sortByString.equals("") ? getPosts(finalQueryObject) : getPostsSorted(finalQueryObject); + + } + + + public String convertFilterString(String filterParameter) { + // Converts the ?filter= parameter of the URL to one that can be used in the GraphQL query + // I could basically remove the last "s" and call toUpperCase instead of this switch statement but this looks easier to read. + switch (filterParameter.toLowerCase()) { + case "pictures": + return "PICTURE"; + case "videos": + return "VIDEO"; + case "albums": + return "ALBUM"; + case "": + return "NOFILTER"; + default: + LOGGER.error(String.format("Invalid filter %s using no filter",filterParameter)); + return ""; + } + } + + public String getParameter(URL url, String parameter) throws MalformedURLException { + // Gets passed parameters from the URL + String toReplace = String.format("https://scrolller.com/r/%s?",getGID(url)); + List args= URLEncodedUtils.parse(url.toExternalForm(), Charset.defaultCharset()); + for (NameValuePair arg:args) { + // First parameter contains part of the url so we have to remove it + // Ex: for the url https://scrolller.com/r/CatsStandingUp?filter=xxxx&sort=yyyy + // 1) arg.getName() => https://scrolller.com/r/CatsStandingUp?filter + // 2) arg.getName() => sort + + if (arg.getName().replace(toReplace,"").toLowerCase().equals((parameter))) { + return arg.getValue(); + } + } + return ""; + } + + private JSONObject getPosts(JSONObject data) { + // The actual GraphQL query call + + try { + String url = "https://api.scrolller.com/api/v2/graphql"; + + URL obj = new URL(url); + HttpURLConnection conn = (HttpURLConnection) obj.openConnection(); + conn.setReadTimeout(5000); + conn.addRequestProperty("Accept-Language", "en-US,en;q=0.8"); + conn.addRequestProperty("User-Agent", "Mozilla"); + conn.addRequestProperty("Referer", "scrolller.com"); + + conn.setDoOutput(true); + + OutputStreamWriter w = new OutputStreamWriter(conn.getOutputStream(), "UTF-8"); + + w.write(data.toString()); + w.close(); + + BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream())); + String inputLine; + StringBuffer jsonString = new StringBuffer(); + + while ((inputLine = in.readLine()) != null) { + jsonString.append(inputLine); + } + + in.close(); + conn.disconnect(); + + return new JSONObject(jsonString.toString()); + + } catch (Exception e) { + e.printStackTrace(); + } + + return new JSONObject("{}"); + } + + private JSONObject getPostsSorted(JSONObject data) throws MalformedURLException { + + // The actual GraphQL query call (if sort parameter is present) + try { + + ArrayList postsJsonStrings = new ArrayList<>(); + + WebSocketClient wsc = new WebSocketClient(new URI("wss://api.scrolller.com/api/v2/graphql")) { + @Override + public void onOpen(ServerHandshake serverHandshake) { + // As soon as the WebSocket connects send our query + this.send(data.toString()); + } + + @Override + public void onMessage(String s) { + postsJsonStrings.add(s); + if (new JSONObject(s).getJSONObject("data").getJSONObject("fetchSubreddit").has("iterator")) { + this.close(); + } + } + + @Override + public void onClose(int i, String s, boolean b) { + } + + @Override + public void onError(Exception e) { + LOGGER.error(String.format("WebSocket error, server reported %s", e.getMessage())); + } + }; + wsc.connect(); + + while (!wsc.isClosed()) { + // Posts list is not over until the connection closes. + } + + JSONObject finalObject = new JSONObject(); + JSONArray posts = new JSONArray(); + + // Iterator is the last object in the post list, let's duplicate it in his own object for clarity. + finalObject.put("iterator", new JSONObject(postsJsonStrings.get(postsJsonStrings.size()-1))); + + for (String postString : postsJsonStrings) { + posts.put(new JSONObject(postString)); + } + finalObject.put("posts", posts); + + if (finalObject.getJSONArray("posts").length() == 1 && !finalObject.getJSONArray("posts").getJSONObject(0).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { + // Only iterator, no posts. + return null; + } + + return finalObject; + + + } catch (URISyntaxException ue) { + // Nothing to catch, it's an hardcoded URI. + } + + return null; + } + + + @Override + protected List getURLsFromJSON(JSONObject json) throws JSONException { + + boolean sortRequested = json.has("posts"); + + int bestArea = 0; + String bestUrl = ""; + List list = new ArrayList<>(); + + JSONArray itemsList = sortRequested ? json.getJSONArray("posts") : json.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").getJSONArray("items"); + + for (Object item : itemsList) { + + if (sortRequested && !((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { + continue; + } + + JSONArray sourcesTMP = sortRequested ? ((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").getJSONArray("mediaSources") : ((JSONObject) item).getJSONArray("mediaSources"); + for (Object sourceTMP : sourcesTMP) + { + int widthTMP = ((JSONObject) sourceTMP).getInt("width"); + int heightTMP = ((JSONObject) sourceTMP).getInt("height"); + int areaTMP = widthTMP * heightTMP; + + if (areaTMP > bestArea) { + bestArea = widthTMP; + bestUrl = ((JSONObject) sourceTMP).getString("url"); + } + } + list.add(bestUrl); + bestUrl = ""; + bestArea = 0; + } + + return list; + } + + @Override + protected JSONObject getFirstPage() throws IOException { + try { + return prepareQuery(null, this.getGID(url), getParameter(url,"sort")); + } catch (URISyntaxException e) { + LOGGER.error(String.format("Error obtaining first page: %s", e.getMessage())); + return null; + } + } + + @Override + public JSONObject getNextPage(JSONObject source) throws IOException { + // Every call the the API contains an "iterator" string that we need to pass to the API to get the next page + // Checking if iterator is null is not working for some reason, hence why the weird "iterator.toString().equals("null")" + + Object iterator = null; + if (source.has("iterator")) { + // Sort requested, custom JSON. + iterator = source.getJSONObject("iterator").getJSONObject("data").getJSONObject("fetchSubreddit").get("iterator"); + } else { + iterator = source.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").get("iterator"); + } + + if (!iterator.toString().equals("null")) { + // Need to change page. + try { + return prepareQuery(iterator.toString(), this.getGID(url), getParameter(url,"sort")); + } catch (URISyntaxException e) { + LOGGER.error(String.format("Error changing page: %s", e.getMessage())); + return null; + } + } else { + return null; + } + } +} \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java new file mode 100644 index 00000000..c7bf3d7d --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java @@ -0,0 +1,53 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.ScrolllerRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; + +public class ScrolllerRipperTest extends RippersTest { + @Test + public void testScrolllerGID() throws IOException { + Map testURLs = new HashMap<>(); + + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp"), "CatsStandingUp"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=pictures"), "CatsStandingUp"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?sort=top&filter=pictures"), "CatsStandingUp"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=pictures&sort=top"), "CatsStandingUp"); + for (URL url : testURLs.keySet()) { + ScrolllerRipper ripper = new ScrolllerRipper(url); + ripper.setup(); + Assertions.assertEquals(testURLs.get(url), ripper.getGID(ripper.getURL())); + deleteDir(ripper.getWorkingDir()); + } + } + + @Test + public void testScrolllerFilterRegex() throws IOException { + Map testURLs = new HashMap<>(); + + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp"), "NOFILTER"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=pictures"), "PICTURE"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=videos"), "VIDEO"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=albums"), "ALBUM"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?sort=top&filter=pictures"), "PICTURE"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?sort=top&filter=videos"), "VIDEO"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?sort=top&filter=albums"), "ALBUM"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=pictures&sort=top"), "PICTURE"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=videos&sort=top"), "VIDEO"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=albums&sort=top"), "ALBUM"); + for (URL url : testURLs.keySet()) { + ScrolllerRipper ripper = new ScrolllerRipper(url); + ripper.setup(); + Assertions.assertEquals(testURLs.get(url), ripper.convertFilterString(ripper.getParameter(ripper.getURL(),"filter"))); + deleteDir(ripper.getWorkingDir()); + } + } + + + +} From 8609f68f256684fe0cb190800a4e4437d15961ff Mon Sep 17 00:00:00 2001 From: xuxuck Date: Wed, 12 Aug 2020 14:37:09 +0200 Subject: [PATCH 110/512] Folder names remain closer to their original name. Fixes #1258 --- src/main/java/com/rarchives/ripme/utils/Utils.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index a009c7a1..ba2a799b 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -486,8 +486,15 @@ public class Utils { return text; } + /** + * Removes any potentially unsafe characters from a string and truncates it on a maximum length of 100 characters. + * Characters considered safe are alpha numerical characters as well as minus, dot, comma, underscore and whitespace. + * + * @param text The potentially unsafe text + * @return a filesystem safe string + */ public static String filesystemSafe(String text) { - text = text.replaceAll("[^a-zA-Z0-9.-]", "_").replaceAll("__", "_").replaceAll("_+$", ""); + text = text.replaceAll("[^a-zA-Z0-9-.,_ ]", ""); if (text.length() > 100) { text = text.substring(0, 99); } From ed157dc80be5dbcf2d54bdf9b0d9f18711492d6c Mon Sep 17 00:00:00 2001 From: borderline232 Date: Fri, 8 Jan 2021 11:05:56 -0500 Subject: [PATCH 111/512] Added Soundgasm Ripper - Created new ripper to scrape soundgasm.net urls and obtain their audio - Added ripper to RipUtils so its deteced in the reddit ripper - Created test for basic and reddit urls and both passed --- .../ripme/ripper/rippers/SoundgasmRipper.java | 68 +++++++++++++++++++ .../com/rarchives/ripme/utils/RipUtils.java | 15 ++++ .../ripper/rippers/SoundgasmRipperTest.java | 23 +++++++ 3 files changed, 106 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java new file mode 100644 index 00000000..65a1f1de --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java @@ -0,0 +1,68 @@ +package com.rarchives.ripme.ripper.rippers; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class SoundgasmRipper extends AbstractHTMLRipper { + + private static final String HOST = "soundgasm.net"; + + public SoundgasmRipper(URL url) throws IOException { + super(new URL(url.toExternalForm())); + } + + @Override + protected String getDomain() { return "soundgasm.net"; } + + @Override + public String getHost() { return "soundgasm"; } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^/u/([a-zA-Z0-9_-]+)/([a-zA-Z0-9_-]+).*$"); + Matcher m = p.matcher(url.getFile()); + if (m.find()) { + return m.group(m.groupCount()); + } + throw new MalformedURLException( + "Expected soundgasm.net format: " + + "soundgasm.net/u/username/id or " + + " Got: " + url); + } + + @Override + public Document getFirstPage() throws IOException { + return Http.url(url).get(); + } + + @Override + public List getURLsFromPage(Document page) { + List res = new ArrayList<>(); + + Elements script = page.select("script"); + Pattern p = Pattern.compile("m4a\\:\\s\"(https?:.*)\\\""); + + for (Element e: script) { + Matcher m = p.matcher(e.data()); + if (m.find()) { res.add(m.group(1)); } + } + return res; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + +} diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 3fcb71c2..503a8165 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -14,6 +14,7 @@ import com.rarchives.ripme.ripper.rippers.ImgurRipper; import com.rarchives.ripme.ripper.rippers.RedgifsRipper; import com.rarchives.ripme.ripper.rippers.VidbleRipper; import com.rarchives.ripme.ripper.rippers.GfycatRipper; +import com.rarchives.ripme.ripper.rippers.SoundgasmRipper; import org.apache.commons.lang.math.NumberUtils; import org.apache.log4j.Logger; import org.jsoup.Jsoup; @@ -127,6 +128,20 @@ public class RipUtils { } return result; } + else if (url.toExternalForm().contains("soundgasm.net")) { + try { + logger.info("Getting soundgasm page " + url); + SoundgasmRipper r = new SoundgasmRipper(url); + Document tempDoc = r.getFirstPage(); + for (String u : r.getURLsFromPage(tempDoc)) { + result.add(new URL(u)); + } + } catch (IOException e) { + // Do nothing + logger.warn("Exception while retrieving soundgasm page:", e); + } + return result; + } Pattern p = Pattern.compile("https?://i.reddituploads.com/([a-zA-Z0-9]+)\\?.*"); Matcher m = p.matcher(url.toExternalForm()); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java new file mode 100644 index 00000000..753e7b78 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java @@ -0,0 +1,23 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.RedditRipper; +import org.junit.Test; +import com.rarchives.ripme.ripper.rippers.SoundgasmRipper; + +import java.io.IOException; +import java.net.URL; + +public class SoundgasmRipperTest extends RippersTest { + + @Test + public void testSoundgasmURLs() throws IOException { + SoundgasmRipper ripper = new SoundgasmRipper(new URL("https://soundgasm.net/u/_Firefly_xoxo/Rambles-with-my-Lovense")); + testRipper(ripper); + } + + @Test + public void testRedditSoundgasmURL() throws IOException { + RedditRipper ripper = new RedditRipper(new URL("https://www.reddit.com/r/gonewildaudio/comments/kn1bvj/f4m_mistress_controlled_my_lovense_while_i_tried/")); + testRipper(ripper); + } +} From 83ef11c977242d6df677682d0f37eec2e84b0af3 Mon Sep 17 00:00:00 2001 From: Omar Morales Date: Tue, 8 Dec 2020 10:43:08 -0600 Subject: [PATCH 112/512] NullPointerException handled for invalid save path. #1785 --- .../com/rarchives/ripme/ripper/DownloadFileThread.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index ba1104eb..c05fe0f9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -282,7 +282,14 @@ class DownloadFileThread extends Thread { logger.debug("IOException", e); logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + e.getMessage()); - } finally { + } catch (NullPointerException npe){ + + logger.error("[!] " + Utils.getLocalizedString("failed.to.download") + " for URL " + url); + observer.downloadErrored(url, + Utils.getLocalizedString("failed.to.download") + " " + url.toExternalForm()); + return; + + }finally { // Close any open streams try { if (bis != null) { From 2a509640cf8922599dbacfd999353466f76c7cc2 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 23 Jan 2021 20:19:28 +0100 Subject: [PATCH 113/512] imagebam, motherless tests flaky --- .../rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java | 2 ++ .../ripme/tst/ripper/rippers/MotherlessRipperTest.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java index efe57b96..5ecfe3f6 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ImagebamRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class ImagebamRipperTest extends RippersTest { @Test + @Tag("flaky") public void testImagebamRip() throws IOException { ImagebamRipper ripper = new ImagebamRipper(new URL("http://www.imagebam.com/gallery/488cc796sllyf7o5srds8kpaz1t4m78i")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java index 2739f9da..97f48a5f 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java @@ -5,10 +5,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.MotherlessRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class MotherlessRipperTest extends RippersTest { @Test + @Tag("flaky") public void testMotherlessAlbumRip() throws IOException { MotherlessRipper ripper = new MotherlessRipper(new URL("https://motherless.com/G1168D90")); testRipper(ripper); From 0f83a2a8f6c2c7bc2e9238a690de42ab6b420ad6 Mon Sep 17 00:00:00 2001 From: PaaaulZ <46759927+PaaaulZ@users.noreply.github.com> Date: Mon, 25 Jan 2021 02:44:56 +0100 Subject: [PATCH 114/512] Added support for Cyberdrop.me (fix #1746) --- .../ripme/ripper/rippers/CyberdropRipper.java | 60 +++++++++++++++++++ .../ripper/rippers/CyberdropRipperTest.java | 51 ++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java new file mode 100644 index 00000000..f288592a --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java @@ -0,0 +1,60 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.*; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +public class CyberdropRipper extends AbstractHTMLRipper { + + public CyberdropRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "cyberdrop"; + } + + @Override + protected Document getFirstPage() throws IOException { + return Http.url(url).get(); + } + + @Override + public String getDomain() { + return "cyberdrop.me"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^https?://cyberdrop\\.me/a/([a-zA-Z0-9]+).*?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected cyberdrop.me URL format: " + + "https://cyberdrop.me/a/xxxxxxxx - got " + url + "instead"); + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + + @Override + protected List getURLsFromPage(Document page) { + ArrayList urls = new ArrayList<>(); + for (Element element: page.getElementsByClass("image")) { + urls.add(element.attr("href")); + } + return urls; + } +} \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java new file mode 100644 index 00000000..4d077628 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java @@ -0,0 +1,51 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.CyberdropRipper; +import com.rarchives.ripme.utils.Http; +import org.jsoup.nodes.Document; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class CyberdropRipperTest extends RippersTest { + @Test + public void testScrolllerGID() throws IOException { + Map testURLs = new HashMap<>(); + + testURLs.put(new URL("https://cyberdrop.me/a/n4umdBjw"), "n4umdBjw"); + testURLs.put(new URL("https://cyberdrop.me/a/iLtp4BjW"), "iLtp4BjW"); + for (URL url : testURLs.keySet()) { + CyberdropRipper ripper = new CyberdropRipper(url); + ripper.setup(); + Assertions.assertEquals(testURLs.get(url), ripper.getGID(ripper.getURL())); + deleteDir(ripper.getWorkingDir()); + } + } + + @Test + public void testCyberdropNumberOfFiles() throws IOException { + List testURLs = new ArrayList(); + + testURLs.add(new URL("https://cyberdrop.me/a/n4umdBjw")); + testURLs.add(new URL("https://cyberdrop.me/a/iLtp4BjW")); + for (URL url : testURLs) { + Assertions.assertTrue(willDownloadAllFiles(url)); + } + } + + public boolean willDownloadAllFiles(URL url) throws IOException { + Document doc = Http.url(url).get(); + long numberOfLinks = doc.getElementsByClass("image").stream().count(); + int numberOfFiles = Integer.parseInt(doc.getElementById("totalFilesAmount").text()); + return numberOfLinks == numberOfFiles; + } + + + +} \ No newline at end of file From be4e859eae504d96d4fa26aec2f914937752e311 Mon Sep 17 00:00:00 2001 From: PaaaulZ <46759927+PaaaulZ@users.noreply.github.com> Date: Mon, 25 Jan 2021 02:44:56 +0100 Subject: [PATCH 115/512] Added support for Cyberdrop.me (fix #1746) --- .../ripme/ripper/rippers/CyberdropRipper.java | 60 +++++++++++++++++++ .../ripper/rippers/CyberdropRipperTest.java | 51 ++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java new file mode 100644 index 00000000..f288592a --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java @@ -0,0 +1,60 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.*; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +public class CyberdropRipper extends AbstractHTMLRipper { + + public CyberdropRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "cyberdrop"; + } + + @Override + protected Document getFirstPage() throws IOException { + return Http.url(url).get(); + } + + @Override + public String getDomain() { + return "cyberdrop.me"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^https?://cyberdrop\\.me/a/([a-zA-Z0-9]+).*?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected cyberdrop.me URL format: " + + "https://cyberdrop.me/a/xxxxxxxx - got " + url + "instead"); + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + + @Override + protected List getURLsFromPage(Document page) { + ArrayList urls = new ArrayList<>(); + for (Element element: page.getElementsByClass("image")) { + urls.add(element.attr("href")); + } + return urls; + } +} \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java new file mode 100644 index 00000000..4d077628 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java @@ -0,0 +1,51 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.CyberdropRipper; +import com.rarchives.ripme.utils.Http; +import org.jsoup.nodes.Document; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class CyberdropRipperTest extends RippersTest { + @Test + public void testScrolllerGID() throws IOException { + Map testURLs = new HashMap<>(); + + testURLs.put(new URL("https://cyberdrop.me/a/n4umdBjw"), "n4umdBjw"); + testURLs.put(new URL("https://cyberdrop.me/a/iLtp4BjW"), "iLtp4BjW"); + for (URL url : testURLs.keySet()) { + CyberdropRipper ripper = new CyberdropRipper(url); + ripper.setup(); + Assertions.assertEquals(testURLs.get(url), ripper.getGID(ripper.getURL())); + deleteDir(ripper.getWorkingDir()); + } + } + + @Test + public void testCyberdropNumberOfFiles() throws IOException { + List testURLs = new ArrayList(); + + testURLs.add(new URL("https://cyberdrop.me/a/n4umdBjw")); + testURLs.add(new URL("https://cyberdrop.me/a/iLtp4BjW")); + for (URL url : testURLs) { + Assertions.assertTrue(willDownloadAllFiles(url)); + } + } + + public boolean willDownloadAllFiles(URL url) throws IOException { + Document doc = Http.url(url).get(); + long numberOfLinks = doc.getElementsByClass("image").stream().count(); + int numberOfFiles = Integer.parseInt(doc.getElementById("totalFilesAmount").text()); + return numberOfLinks == numberOfFiles; + } + + + +} \ No newline at end of file From a55f718c84bee0b614d9ed0861b5b714deb3312b Mon Sep 17 00:00:00 2001 From: vlad doster Date: Wed, 27 Jan 2021 15:18:51 -0600 Subject: [PATCH 116/512] (docs) update README.md Use new brew syntax --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b2525c5..3094e1a6 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ For information about running the `.jar` file, see [the How To Run wiki](https:/ On macOS, there is a [cask](https://github.com/Homebrew/homebrew-cask/blob/master/Casks/ripme.rb). ``` -brew cask install ripme && xattr -d com.apple.quarantine /Applications/ripme.jar +brew install --cask ripme && xattr -d com.apple.quarantine /Applications/ripme.jar ``` ## Changelog From e66e46ed2fcc2f368cd6b277a55d9b9044886f66 Mon Sep 17 00:00:00 2001 From: vlad doster Date: Wed, 27 Jan 2021 15:18:51 -0600 Subject: [PATCH 117/512] (docs) update README.md Use new brew syntax --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b2525c5..3094e1a6 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ For information about running the `.jar` file, see [the How To Run wiki](https:/ On macOS, there is a [cask](https://github.com/Homebrew/homebrew-cask/blob/master/Casks/ripme.rb). ``` -brew cask install ripme && xattr -d com.apple.quarantine /Applications/ripme.jar +brew install --cask ripme && xattr -d com.apple.quarantine /Applications/ripme.jar ``` ## Changelog From 48370a0f63491594b026b5b7cd7a72e1a96c66a7 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 30 Jan 2021 11:05:15 +0100 Subject: [PATCH 118/512] test with java-15 --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 19b99691..ea6ac25a 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -12,7 +12,7 @@ jobs: java: [1.8] include: # test newest java on one os only, upload from ubuntu java8 - os: ubuntu-latest - java: 1.14 + java: 1.15 - os: ubuntu-latest upload: true From b94f64652d456ac411da9d94b07cff87c41501d1 Mon Sep 17 00:00:00 2001 From: saladzic Date: Mon, 8 Feb 2021 04:39:54 +0100 Subject: [PATCH 119/512] Fix #1830 + add config option "erome.laravel_session" for EromeRipper to avoid JS-Browser-Check --- .../ripme/ripper/rippers/EromeRipper.java | 77 ++++++++++++------- 1 file changed, 48 insertions(+), 29 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java index 9b586b9a..b44d34d4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.rarchives.ripme.utils.Utils; import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -23,7 +25,7 @@ import com.rarchives.ripme.utils.Http; public class EromeRipper extends AbstractHTMLRipper { boolean rippingProfile; - + private HashMap cookies = new HashMap<>(); public EromeRipper (URL url) throws IOException { super(url); @@ -31,12 +33,12 @@ public class EromeRipper extends AbstractHTMLRipper { @Override public String getDomain() { - return "erome.com"; + return "erome.com"; } @Override public String getHost() { - return "erome"; + return "erome"; } @Override @@ -67,19 +69,19 @@ public class EromeRipper extends AbstractHTMLRipper { @Override public String getAlbumTitle(URL url) throws MalformedURLException { - try { - // Attempt to use album title as GID - Element titleElement = getFirstPage().select("meta[property=og:title]").first(); - String title = titleElement.attr("content"); - title = title.substring(title.lastIndexOf('/') + 1); - return getHost() + "_" + getGID(url) + "_" + title.trim(); - } catch (IOException e) { - // Fall back to default album naming convention - LOGGER.info("Unable to find title at " + url); - } catch (NullPointerException e) { - return getHost() + "_" + getGID(url); - } - return super.getAlbumTitle(url); + try { + // Attempt to use album title as GID + Element titleElement = getFirstPage().select("meta[property=og:title]").first(); + String title = titleElement.attr("content"); + title = title.substring(title.lastIndexOf('/') + 1); + return getHost() + "_" + getGID(url) + "_" + title.trim(); + } catch (IOException e) { + // Fall back to default album naming convention + LOGGER.info("Unable to find title at " + url); + } catch (NullPointerException e) { + return getHost() + "_" + getGID(url); + } + return super.getAlbumTitle(url); } @Override @@ -96,9 +98,11 @@ public class EromeRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { + this.setAuthCookie(); Response resp = Http.url(this.url) - .ignoreContentType() - .response(); + .cookies(cookies) + .ignoreContentType() + .response(); return resp.parse(); } @@ -124,18 +128,18 @@ public class EromeRipper extends AbstractHTMLRipper { private List getMediaFromPage(Document doc) { List results = new ArrayList<>(); for (Element el : doc.select("img.img-front")) { - if (el.hasAttr("src")) { - if (el.attr("src").startsWith("https:")) { - results.add(el.attr("src")); - } else { - results.add("https:" + el.attr("src")); - } - } else if (el.hasAttr("data-src")) { - //to add images that are not loaded( as all images are lasyloaded as we scroll). - results.add(el.attr("data-src")); - } + if (el.hasAttr("src")) { + if (el.attr("src").startsWith("https:")) { + results.add(el.attr("src")); + } else { + results.add("https:" + el.attr("src")); + } + } else if (el.hasAttr("data-src")) { + //to add images that are not loaded( as all images are lasyloaded as we scroll). + results.add(el.attr("data-src")); + } - } + } for (Element el : doc.select("source[label=HD]")) { if (el.attr("src").startsWith("https:")) { results.add(el.attr("src")); @@ -152,7 +156,22 @@ public class EromeRipper extends AbstractHTMLRipper { results.add("https:" + el.attr("src")); } } + + if (results.size() == 0) { + if (cookies.isEmpty()) { + LOGGER.warn("You might try setting erome.laravel_session manually " + + "if you think this page definitely contains media."); + } + } + return results; } + private void setAuthCookie() { + String sessionId = Utils.getConfigString("erome.laravel_session", null); + if (sessionId != null) { + cookies.put("laravel_session", sessionId); + } + } + } From 9330b03eb7a3eae44c45136c84ad45c5dabb8f28 Mon Sep 17 00:00:00 2001 From: saladzic Date: Mon, 8 Feb 2021 04:39:54 +0100 Subject: [PATCH 120/512] Fix #1830 + add config option "erome.laravel_session" for EromeRipper to avoid JS-Browser-Check --- .../ripme/ripper/rippers/EromeRipper.java | 77 ++++++++++++------- 1 file changed, 48 insertions(+), 29 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java index 9b586b9a..b44d34d4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.rarchives.ripme.utils.Utils; import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -23,7 +25,7 @@ import com.rarchives.ripme.utils.Http; public class EromeRipper extends AbstractHTMLRipper { boolean rippingProfile; - + private HashMap cookies = new HashMap<>(); public EromeRipper (URL url) throws IOException { super(url); @@ -31,12 +33,12 @@ public class EromeRipper extends AbstractHTMLRipper { @Override public String getDomain() { - return "erome.com"; + return "erome.com"; } @Override public String getHost() { - return "erome"; + return "erome"; } @Override @@ -67,19 +69,19 @@ public class EromeRipper extends AbstractHTMLRipper { @Override public String getAlbumTitle(URL url) throws MalformedURLException { - try { - // Attempt to use album title as GID - Element titleElement = getFirstPage().select("meta[property=og:title]").first(); - String title = titleElement.attr("content"); - title = title.substring(title.lastIndexOf('/') + 1); - return getHost() + "_" + getGID(url) + "_" + title.trim(); - } catch (IOException e) { - // Fall back to default album naming convention - LOGGER.info("Unable to find title at " + url); - } catch (NullPointerException e) { - return getHost() + "_" + getGID(url); - } - return super.getAlbumTitle(url); + try { + // Attempt to use album title as GID + Element titleElement = getFirstPage().select("meta[property=og:title]").first(); + String title = titleElement.attr("content"); + title = title.substring(title.lastIndexOf('/') + 1); + return getHost() + "_" + getGID(url) + "_" + title.trim(); + } catch (IOException e) { + // Fall back to default album naming convention + LOGGER.info("Unable to find title at " + url); + } catch (NullPointerException e) { + return getHost() + "_" + getGID(url); + } + return super.getAlbumTitle(url); } @Override @@ -96,9 +98,11 @@ public class EromeRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { + this.setAuthCookie(); Response resp = Http.url(this.url) - .ignoreContentType() - .response(); + .cookies(cookies) + .ignoreContentType() + .response(); return resp.parse(); } @@ -124,18 +128,18 @@ public class EromeRipper extends AbstractHTMLRipper { private List getMediaFromPage(Document doc) { List results = new ArrayList<>(); for (Element el : doc.select("img.img-front")) { - if (el.hasAttr("src")) { - if (el.attr("src").startsWith("https:")) { - results.add(el.attr("src")); - } else { - results.add("https:" + el.attr("src")); - } - } else if (el.hasAttr("data-src")) { - //to add images that are not loaded( as all images are lasyloaded as we scroll). - results.add(el.attr("data-src")); - } + if (el.hasAttr("src")) { + if (el.attr("src").startsWith("https:")) { + results.add(el.attr("src")); + } else { + results.add("https:" + el.attr("src")); + } + } else if (el.hasAttr("data-src")) { + //to add images that are not loaded( as all images are lasyloaded as we scroll). + results.add(el.attr("data-src")); + } - } + } for (Element el : doc.select("source[label=HD]")) { if (el.attr("src").startsWith("https:")) { results.add(el.attr("src")); @@ -152,7 +156,22 @@ public class EromeRipper extends AbstractHTMLRipper { results.add("https:" + el.attr("src")); } } + + if (results.size() == 0) { + if (cookies.isEmpty()) { + LOGGER.warn("You might try setting erome.laravel_session manually " + + "if you think this page definitely contains media."); + } + } + return results; } + private void setAuthCookie() { + String sessionId = Utils.getConfigString("erome.laravel_session", null); + if (sessionId != null) { + cookies.put("laravel_session", sessionId); + } + } + } From 8a86139759434d10044dc7fed542798ce1f84b8a Mon Sep 17 00:00:00 2001 From: saladzic <69090391+saladzic@users.noreply.github.com> Date: Fri, 12 Feb 2021 22:07:36 +0100 Subject: [PATCH 121/512] Fix #1837 (Xhamster Junit tests) + added support for domain endings "one" and "desi" --- .../ripme/ripper/rippers/XhamsterRipper.java | 71 ++++++++++++------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 7ae570f3..290eee03 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -57,23 +57,23 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.com/photos/gallery/.*?(\\d+)$"); + Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.(com|one|desi)/photos/gallery/.*?(\\d+)$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); } - p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.com/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); + p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return "user_" + m.group(1); } - p = Pattern.compile("^https?://.*xhamster2?\\.com/(movies|videos)/(.*)$"); + p = Pattern.compile("^https?://.*xhamster2?\\.(com|one|desi)/(movies|videos)/(.*)$"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(2); } - throw new MalformedURLException( + throw new MalformedURLException( "Expected xhamster.com gallery formats: " + "xhamster.com/photos/gallery/xxxxx-#####" + " Got: " + url); @@ -97,7 +97,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public boolean pageContainsAlbums(URL url) { - Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster\\.com/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); + Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); Matcher m = p.matcher(url.toExternalForm()); LOGGER.info("Checking if page has albums"); LOGGER.info(m.matches()); @@ -139,8 +139,9 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException { - if (doc.select("a[data-page=next]").first() != null) { - String nextPageUrl = doc.select("a[data-page=next]").first().attr("href"); + if (doc.select("a.prev-next-list-link").first() != null) { + String nextPageUrl = doc.select("a.prev-next-list-link").first().attr("href"); + System.out.println(nextPageUrl); if (nextPageUrl.startsWith("http")) { nextPageUrl = nextPageUrl.replaceAll("https?://\\w?\\w?\\.?xhamster\\.", "https://m.xhamster."); nextPageUrl = nextPageUrl.replaceAll("https?://xhamster2\\.", "https://m.xhamster2."); @@ -156,25 +157,45 @@ public class XhamsterRipper extends AbstractHTMLRipper { LOGGER.debug("Checking for urls"); List result = new ArrayList<>(); if (!isVideoUrl(url)) { - for (Element page : doc.select("div.picture_view > div.pictures_block > div.items > div.item-container > a.item")) { - // Make sure we don't waste time running the loop if the ripper has been stopped - if (isStopped()) { - break; - } - String pageWithImageUrl = page.attr("href"); - try { - // This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to - // the page chamster.com but displays the mobile site from m.xhamster.com - pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster\\.", "://m.xhamster."); - pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster2\\.", "://m.xhamster."); - String image = Http.url(new URL(pageWithImageUrl)).get().select("a > img#photoCurr").attr("src"); - downloadFile(image); - } catch (IOException e) { - LOGGER.error("Was unable to load page " + pageWithImageUrl); - } - } + if (!doc.select("div.picture_view > div.pictures_block > div.items > div.item-container > a.item").isEmpty()) { + // Old HTML structure is still present at some places + for (Element page : doc.select("div.picture_view > div.pictures_block > div.items > div.item-container > a.item")) { + // Make sure we don't waste time running the loop if the ripper has been stopped + if (isStopped()) { + break; + } + String pageWithImageUrl = page.attr("href"); + try { + // This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to + // the page chamster.com but displays the mobile site from m.xhamster.com + pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster\\.", "://m.xhamster."); + pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster2\\.", "://m.xhamster."); + String image = Http.url(new URL(pageWithImageUrl)).get().select("a > img#photoCurr").attr("src"); + result.add(image); + downloadFile(image); + } catch (IOException e) { + LOGGER.error("Was unable to load page " + pageWithImageUrl); + } + } + } else { + // New HTML structure + for (Element page : doc.select("div#photo-slider > div#photo_slider > a")) { + // Make sure we don't waste time running the loop if the ripper has been stopped + if (isStopped()) { + break; + } + String image = page.attr("href"); + // This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to + // the page chamster.com but displays the mobile site from m.xhamster.com + image = image.replaceAll("://xhamster\\.", "://m.xhamster."); + image = image.replaceAll("://xhamster2\\.", "://m.xhamster."); + result.add(image); + downloadFile(image); + } + } } else { String imgUrl = doc.select("div.player-container > a").attr("href"); + result.add(imgUrl); downloadFile(imgUrl); } return result; @@ -193,7 +214,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { LOGGER.error("The url \"" + url + "\" is malformed"); } } - + @Override public String getAlbumTitle(URL url) throws MalformedURLException { try { From 00c7eacf9324b90eaf5c47e8b90ace8788f8bfa8 Mon Sep 17 00:00:00 2001 From: Stefan Aladzic <69090391+saladzic@users.noreply.github.com> Date: Mon, 15 Feb 2021 00:49:29 +0100 Subject: [PATCH 122/512] Add Flaky-Tag to testXhamster2Album() and testXhamsterAlbumOneDomain() --- .../rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java index e9475a1e..aaccf47c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java @@ -18,6 +18,7 @@ public class XhamsterRipperTest extends RippersTest { testRipper(ripper); } @Test + @Tag("flaky") public void testXhamster2Album() throws IOException { XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster2.com/photos/gallery/sexy-preggo-girls-9026608")); testRipper(ripper); @@ -28,6 +29,7 @@ public class XhamsterRipperTest extends RippersTest { testRipper(ripper); } @Test + @Tag("flaky") public void testXhamsterAlbumOneDomain() throws IOException { XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.one/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664")); testRipper(ripper); From c14d9b51eeb60ba513b9643c97a293be77474258 Mon Sep 17 00:00:00 2001 From: Stefan Aladzic <69090391+saladzic@users.noreply.github.com> Date: Mon, 15 Feb 2021 01:32:42 +0100 Subject: [PATCH 123/512] The website may be unavailable. Unit tests have been temporarily disabled. --- .../ripme/tst/ripper/rippers/PorncomixDotOneRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixDotOneRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixDotOneRipperTest.java index 9d4df122..1edf7b80 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixDotOneRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixDotOneRipperTest.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.PorncomixDotOneRipper; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class PorncomixDotOneRipperTest extends RippersTest { @Test + @Disabled("website down?") public void testPorncomixAlbum() throws IOException { PorncomixDotOneRipper ripper = new PorncomixDotOneRipper(new URL("https://www.porncomix.one/gallery/blacknwhite-make-america-great-again")); testRipper(ripper); From 5e44215cd1e2cd509416206dd025beb5b96a0dcd Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 13 Feb 2021 12:19:06 +0100 Subject: [PATCH 124/512] remove not necessary parts in Utils.java --- .../java/com/rarchives/ripme/utils/Utils.java | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index a009c7a1..ad78af2f 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -2,7 +2,6 @@ package com.rarchives.ripme.utils; import java.io.File; import java.io.FileNotFoundException; -import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; @@ -22,7 +21,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; @@ -32,7 +30,6 @@ import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Stream; import javax.sound.sampled.AudioSystem; import javax.sound.sampled.Clip; @@ -359,7 +356,7 @@ public class Utils { if (wasFirstParam) { c = "?"; } - url = url.substring(0, paramIndex) + c + url.substring(nextParam + 1, url.length()); + url = url.substring(0, paramIndex) + c + url.substring(nextParam + 1); } else { url = url.substring(0, paramIndex); } @@ -510,8 +507,7 @@ public class Utils { return path; } - String original = path; // needs to be checked if lowercase exists - String lastPart = original.substring(index + 1).toLowerCase(); // setting lowercase to check if it exists + String lastPart = path.substring(index + 1).toLowerCase(); // setting lowercase to check if it exists // Get a List of all Directories and check its lowercase // if file exists return it @@ -525,7 +521,8 @@ public class Utils { } } - return original; + // otherwise return original path + return path; } /** @@ -771,7 +768,7 @@ public class Utils { Path myPath; if (uri.getScheme().equals("jar")) { - FileSystem fileSystem = FileSystems.newFileSystem(uri, Collections.emptyMap()); + FileSystem fileSystem = FileSystems.newFileSystem(uri, Collections.emptyMap()); myPath = fileSystem.getPath("/"); } else { myPath = Paths.get(uri).getParent(); @@ -813,7 +810,7 @@ public class Utils { * bar */ public static String getByteStatusText(int completionPercentage, int bytesCompleted, int bytesTotal) { - return String.valueOf(completionPercentage) + "% - " + Utils.bytesToHumanReadable(bytesCompleted) + " / " + return completionPercentage + "% - " + Utils.bytesToHumanReadable(bytesCompleted) + " / " + Utils.bytesToHumanReadable(bytesTotal); } @@ -847,7 +844,7 @@ public class Utils { for (File file : listOfFiles) { if (file.isFile()) { - String[] filename = file.getName().split("\\.(?=[^\\.]+$)"); // split filename from it's extension + String[] filename = file.getName().split("\\.(?=[^.]+$)"); // split filename from it's extension if (filename[0].equalsIgnoreCase(fileName)) { return true; } @@ -861,15 +858,11 @@ public class Utils { } public static File shortenSaveAsWindows(String ripsDirPath, String fileName) throws FileNotFoundException { - // int ripDirLength = ripsDirPath.length(); - // int maxFileNameLength = 260 - ripDirLength; - // LOGGER.info(maxFileNameLength); LOGGER.error("The filename " + fileName + " is to long to be saved on this file system."); LOGGER.info("Shortening filename"); String fullPath = ripsDirPath + File.separator + fileName; // How long the path without the file name is int pathLength = ripsDirPath.length(); - int fileNameLength = fileName.length(); if (pathLength == 260) { // We've reached the max length, there's nothing more we can do throw new FileNotFoundException("File path is too long for this OS"); @@ -879,7 +872,6 @@ public class Utils { // file extension String fileExt = saveAsSplit[saveAsSplit.length - 1]; // The max limit for paths on Windows is 260 chars - LOGGER.info(fullPath.substring(0, 259 - pathLength - fileExt.length() + 1) + "." + fileExt); fullPath = fullPath.substring(0, 259 - pathLength - fileExt.length() + 1) + "." + fileExt; LOGGER.info(fullPath); LOGGER.info(fullPath.length()); From 3d24d6d61a2d443d3bfc8dedb1d11be6785cdb76 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 26 Jul 2020 08:17:38 +0200 Subject: [PATCH 125/512] add gradle wrapper --- gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 58910 bytes gradle/wrapper/gradle-wrapper.properties | 5 + gradlew | 185 +++++++++++++++++++++++ gradlew.bat | 104 +++++++++++++ 4 files changed, 294 insertions(+) create mode 100644 gradle/wrapper/gradle-wrapper.jar create mode 100644 gradle/wrapper/gradle-wrapper.properties create mode 100755 gradlew create mode 100644 gradlew.bat diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..62d4c053550b91381bbd28b1afc82d634bf73a8a GIT binary patch literal 58910 zcma&ObC74zk}X`WF59+k+qTVL*+!RbS9RI8Z5v&-ZFK4Nn|tqzcjwK__x+Iv5xL`> zj94dg?X`0sMHx^qXds{;KY)OMg#H>35XgTVfq6#vc9ww|9) z@UMfwUqk)B9p!}NrNqTlRO#i!ALOPcWo78-=iy}NsAr~T8T0X0%G{DhX~u-yEwc29WQ4D zuv2j{a&j?qB4wgCu`zOXj!~YpTNFg)TWoV>DhYlR^Gp^rkOEluvxkGLB?!{fD!T@( z%3cy>OkhbIKz*R%uoKqrg1%A?)uTZD&~ssOCUBlvZhx7XHQ4b7@`&sPdT475?*zWy z>xq*iK=5G&N6!HiZaD{NSNhWL;+>Quw_#ZqZbyglna!Fqn3N!$L`=;TFPrhodD-Q` z1l*=DP2gKJP@)cwI@-M}?M$$$%u~=vkeC%>cwR$~?y6cXx-M{=wdT4|3X(@)a|KkZ z`w$6CNS@5gWS7s7P86L<=vg$Mxv$?)vMj3`o*7W4U~*Nden}wz=y+QtuMmZ{(Ir1D zGp)ZsNiy{mS}Au5;(fYf93rs^xvi(H;|H8ECYdC`CiC&G`zw?@)#DjMc7j~daL_A$ z7e3nF2$TKlTi=mOftyFBt8*Xju-OY@2k@f3YBM)-v8+5_o}M?7pxlNn)C0Mcd@87?+AA4{Ti2ptnYYKGp`^FhcJLlT%RwP4k$ad!ho}-^vW;s{6hnjD0*c39k zrm@PkI8_p}mnT&5I@=O1^m?g}PN^8O8rB`;t`6H+?Su0IR?;8txBqwK1Au8O3BZAX zNdJB{bpQWR@J|e=Z>XSXV1DB{uhr3pGf_tb)(cAkp)fS7*Qv))&Vkbb+cvG!j}ukd zxt*C8&RN}5ck{jkw0=Q7ldUp0FQ&Pb_$M7a@^nf`8F%$ftu^jEz36d#^M8Ia{VaTy z5(h$I)*l3i!VpPMW+XGgzL~fcN?{~1QWu9!Gu0jOWWE zNW%&&by0DbXL&^)r-A*7R@;T$P}@3eOj#gqJ!uvTqBL5bupU91UK#d|IdxBUZAeh1 z>rAI#*Y4jv>uhOh7`S@mnsl0g@1C;k$Z%!d*n8#_$)l}-1&z2kr@M+xWoKR z!KySy-7h&Bf}02%JeXmQGjO3ntu={K$jy$rFwfSV8!zqAL_*&e2|CJ06`4&0+ceI026REfNT>JzAdwmIlKLEr2? zaZ#d*XFUN*gpzOxq)cysr&#6zNdDDPH% zd8_>3B}uA7;bP4fKVdd~Og@}dW#74ceETOE- zlZgQqQfEc?-5ly(Z5`L_CCM!&Uxk5#wgo=OLs-kFHFG*cTZ)$VE?c_gQUW&*!2@W2 z7Lq&_Kf88OCo?BHCtwe*&fu&8PQ(R5&lnYo8%+U73U)Ec2&|A)Y~m7(^bh299REPe zn#gyaJ4%o4>diN3z%P5&_aFUmlKytY$t21WGwx;3?UC}vlxi-vdEQgsKQ;=#sJ#ll zZeytjOad$kyON4XxC}frS|Ybh`Yq!<(IrlOXP3*q86ImyV*mJyBn$m~?#xp;EplcM z+6sez%+K}Xj3$YN6{}VL;BZ7Fi|iJj-ywlR+AP8lq~mnt5p_%VmN{Sq$L^z!otu_u znVCl@FgcVXo510e@5(wnko%Pv+^r^)GRh;>#Z(|#cLnu_Y$#_xG&nvuT+~gzJsoSi zBvX`|IS~xaold!`P!h(v|=>!5gk)Q+!0R1Ge7!WpRP{*Ajz$oGG$_?Ajvz6F0X?809o`L8prsJ*+LjlGfSziO;+ zv>fyRBVx#oC0jGK8$%$>Z;0+dfn8x;kHFQ?Rpi7(Rc{Uq{63Kgs{IwLV>pDK7yX-2 zls;?`h!I9YQVVbAj7Ok1%Y+F?CJa-Jl>1x#UVL(lpzBBH4(6v0^4 z3Tf`INjml5`F_kZc5M#^J|f%7Hgxg3#o}Zwx%4l9yYG!WaYUA>+dqpRE3nw#YXIX%= ziH3iYO~jr0nP5xp*VIa#-aa;H&%>{mfAPPlh5Fc!N7^{!z$;p-p38aW{gGx z)dFS62;V;%%fKp&i@+5x=Cn7Q>H`NofJGXmNeh{sOL+Nk>bQJJBw3K*H_$}%*xJM=Kh;s#$@RBR z|75|g85da@#qT=pD777m$wI!Q8SC4Yw3(PVU53bzzGq$IdGQoFb-c_(iA_~qD|eAy z@J+2!tc{|!8fF;%6rY9`Q!Kr>MFwEH%TY0y>Q(D}xGVJM{J{aGN0drG&|1xO!Ttdw z-1^gQ&y~KS5SeslMmoA$Wv$ly={f}f9<{Gm!8ycp*D9m*5Ef{ymIq!MU01*)#J1_! zM_i4{LYButqlQ>Q#o{~W!E_#(S=hR}kIrea_67Z5{W>8PD>g$f;dTvlD=X@T$8D0;BWkle@{VTd&D5^)U>(>g(jFt4lRV6A2(Te->ooI{nk-bZ(gwgh zaH4GT^wXPBq^Gcu%xW#S#p_&x)pNla5%S5;*OG_T^PhIIw1gXP&u5c;{^S(AC*+$> z)GuVq(FT@zq9;i{*9lEsNJZ)??BbSc5vF+Kdh-kL@`(`l5tB4P!9Okin2!-T?}(w% zEpbEU67|lU#@>DppToestmu8Ce=gz=e#V+o)v)#e=N`{$MI5P0O)_fHt1@aIC_QCv=FO`Qf=Ga%^_NhqGI)xtN*^1n{ z&vgl|TrKZ3Vam@wE0p{c3xCCAl+RqFEse@r*a<3}wmJl-hoJoN<|O2zcvMRl<#BtZ z#}-bPCv&OTw`GMp&n4tutf|er`@#d~7X+);##YFSJ)BitGALu}-N*DJdCzs(cQ?I- z6u(WAKH^NUCcOtpt5QTsQRJ$}jN28ZsYx+4CrJUQ%egH zo#tMoywhR*oeIkS%}%WUAIbM`D)R6Ya&@sZvvUEM7`fR0Ga03*=qaEGq4G7-+30Ck zRkje{6A{`ebq?2BTFFYnMM$xcQbz0nEGe!s%}O)m={`075R0N9KTZ>vbv2^eml>@}722%!r#6Wto}?vNst? zs`IasBtcROZG9+%rYaZe^=5y3chDzBf>;|5sP0!sP(t^= z^~go8msT@|rp8LJ8km?4l?Hb%o10h7(ixqV65~5Y>n_zG3AMqM3UxUNj6K-FUgMT7 z*Dy2Y8Ws+%`Z*~m9P zCWQ8L^kA2$rf-S@qHow$J86t)hoU#XZ2YK~9GXVR|*`f6`0&8j|ss_Ai-x=_;Df^*&=bW$1nc{Gplm zF}VF`w)`5A;W@KM`@<9Bw_7~?_@b{Z`n_A6c1AG#h#>Z$K>gX6reEZ*bZRjCup|0# zQ{XAb`n^}2cIwLTN%5Ix`PB*H^(|5S{j?BwItu+MS`1)VW=TnUtt6{3J!WR`4b`LW z?AD#ZmoyYpL=903q3LSM=&5eNP^dwTDRD~iP=}FXgZ@2WqfdyPYl$9do?wX{RU*$S zgQ{OqXK-Yuf4+}x6P#A*la&^G2c2TC;aNNZEYuB(f25|5eYi|rd$;i0qk7^3Ri8of ziP~PVT_|4$n!~F-B1_Et<0OJZ*e+MN;5FFH`iec(lHR+O%O%_RQhvbk-NBQ+$)w{D+dlA0jxI;z|P zEKW`!X)${xzi}Ww5G&@g0akBb_F`ziv$u^hs0W&FXuz=Ap>SUMw9=M?X$`lgPRq11 zqq+n44qL;pgGO+*DEc+Euv*j(#%;>p)yqdl`dT+Og zZH?FXXt`<0XL2@PWYp|7DWzFqxLK)yDXae&3P*#+f+E{I&h=$UPj;ey9b`H?qe*Oj zV|-qgI~v%&oh7rzICXfZmg$8$B|zkjliQ=e4jFgYCLR%yi!9gc7>N z&5G#KG&Hr+UEfB;M(M>$Eh}P$)<_IqC_WKOhO4(cY@Gn4XF(#aENkp&D{sMQgrhDT zXClOHrr9|POHqlmm+*L6CK=OENXbZ+kb}t>oRHE2xVW<;VKR@ykYq04LM9L-b;eo& zl!QQo!Sw{_$-qosixZJWhciN>Gbe8|vEVV2l)`#5vKyrXc6E`zmH(76nGRdL)pqLb@j<&&b!qJRLf>d`rdz}^ZSm7E;+XUJ ziy;xY&>LM?MA^v0Fu8{7hvh_ynOls6CI;kQkS2g^OZr70A}PU;i^~b_hUYN1*j-DD zn$lHQG9(lh&sDii)ip*{;Sb_-Anluh`=l~qhqbI+;=ZzpFrRp&T+UICO!OoqX@Xr_ z32iJ`xSpx=lDDB_IG}k+GTYG@K8{rhTS)aoN8D~Xfe?ul&;jv^E;w$nhu-ICs&Q)% zZ=~kPNZP0-A$pB8)!`TEqE`tY3Mx^`%O`?EDiWsZpoP`e-iQ#E>fIyUx8XN0L z@S-NQwc;0HjSZKWDL}Au_Zkbh!juuB&mGL0=nO5)tUd_4scpPy&O7SNS^aRxUy0^< zX}j*jPrLP4Pa0|PL+nrbd4G;YCxCK-=G7TG?dby~``AIHwxqFu^OJhyIUJkO0O<>_ zcpvg5Fk$Wpj}YE3;GxRK67P_Z@1V#+pu>pRj0!mFf(m_WR3w3*oQy$s39~U7Cb}p(N&8SEwt+)@%o-kW9Ck=^?tvC2$b9% ze9(Jn+H`;uAJE|;$Flha?!*lJ0@lKfZM>B|c)3lIAHb;5OEOT(2453m!LgH2AX=jK zQ93An1-#l@I@mwB#pLc;M7=u6V5IgLl>E%gvE|}Hvd4-bE1>gs(P^C}gTv*&t>W#+ zASLRX$y^DD3Jrht zwyt`yuA1j(TcP*0p*Xkv>gh+YTLrcN_HuaRMso~0AJg`^nL#52dGBzY+_7i)Ud#X) zVwg;6$WV20U2uyKt8<)jN#^1>PLg`I`@Mmut*Zy!c!zshSA!e^tWVoKJD%jN&ml#{ z@}B$j=U5J_#rc%T7(DGKF+WwIblEZ;Vq;CsG~OKxhWYGJx#g7fxb-_ya*D0=_Ys#f zhXktl=Vnw#Z_neW>Xe#EXT(4sT^3p6srKby4Ma5LLfh6XrHGFGgM;5Z}jv-T!f~=jT&n>Rk z4U0RT-#2fsYCQhwtW&wNp6T(im4dq>363H^ivz#>Sj;TEKY<)dOQU=g=XsLZhnR>e zd}@p1B;hMsL~QH2Wq>9Zb; zK`0`09fzuYg9MLJe~cdMS6oxoAD{kW3sFAqDxvFM#{GpP^NU@9$d5;w^WgLYknCTN z0)N425mjsJTI@#2kG-kB!({*+S(WZ-{SckG5^OiyP%(6DpRsx60$H8M$V65a_>oME z^T~>oG7r!ew>Y)&^MOBrgc-3PezgTZ2xIhXv%ExMFgSf5dQbD=Kj*!J4k^Xx!Z>AW ziZfvqJvtm|EXYsD%A|;>m1Md}j5f2>kt*gngL=enh<>#5iud0dS1P%u2o+>VQ{U%(nQ_WTySY(s#~~> zrTsvp{lTSup_7*Xq@qgjY@1#bisPCRMMHnOL48qi*jQ0xg~TSW%KMG9zN1(tjXix()2$N}}K$AJ@GUth+AyIhH6Aeh7qDgt#t*`iF5#A&g4+ zWr0$h9Zx6&Uo2!Ztcok($F>4NA<`dS&Js%L+67FT@WmI)z#fF~S75TUut%V($oUHw z$IJsL0X$KfGPZYjB9jaj-LaoDD$OMY4QxuQ&vOGo?-*9@O!Nj>QBSA6n$Lx|^ zky)4+sy{#6)FRqRt6nM9j2Lzba!U;aL%ZcG&ki1=3gFx6(&A3J-oo|S2_`*w9zT)W z4MBOVCp}?4nY)1))SOX#6Zu0fQQ7V{RJq{H)S#;sElY)S)lXTVyUXTepu4N)n85Xo zIpWPT&rgnw$D2Fsut#Xf-hO&6uA0n~a;a3!=_!Tq^TdGE&<*c?1b|PovU}3tfiIUu z){4W|@PY}zJOXkGviCw^x27%K_Fm9GuKVpd{P2>NJlnk^I|h2XW0IO~LTMj>2<;S* zZh2uRNSdJM$U$@=`zz}%;ucRx{aKVxxF7?0hdKh6&GxO6f`l2kFncS3xu0Ly{ew0& zeEP*#lk-8-B$LD(5yj>YFJ{yf5zb41PlW7S{D9zC4Aa4nVdkDNH{UsFJp)q-`9OYt zbOKkigbmm5hF?tttn;S4g^142AF^`kiLUC?e7=*JH%Qe>uW=dB24NQa`;lm5yL>Dyh@HbHy-f%6Vz^ zh&MgwYsh(z#_fhhqY$3*f>Ha}*^cU-r4uTHaT?)~LUj5``FcS46oyoI5F3ZRizVD% zPFY(_S&5GN8$Nl2=+YO6j4d|M6O7CmUyS&}m4LSn6}J`$M0ZzT&Ome)ZbJDFvM&}A zZdhDn(*viM-JHf84$!I(8eakl#zRjJH4qfw8=60 z11Ely^FyXjVvtv48-Fae7p=adlt9_F^j5#ZDf7)n!#j?{W?@j$Pi=k`>Ii>XxrJ?$ z^bhh|X6qC8d{NS4rX5P!%jXy=>(P+r9?W(2)|(=a^s^l~x*^$Enw$~u%WRuRHHFan{X|S;FD(Mr z@r@h^@Bs#C3G;~IJMrERd+D!o?HmFX&#i|~q(7QR3f8QDip?ms6|GV_$86aDb|5pc?_-jo6vmWqYi{P#?{m_AesA4xX zi&ki&lh0yvf*Yw~@jt|r-=zpj!bw<6zI3Aa^Wq{|*WEC}I=O!Re!l~&8|Vu<$yZ1p zs-SlwJD8K!$(WWyhZ+sOqa8cciwvyh%zd`r$u;;fsHn!hub0VU)bUv^QH?x30#;tH zTc_VbZj|prj7)d%ORU;Vs{#ERb>K8>GOLSImnF7JhR|g$7FQTU{(a7RHQ*ii-{U3X z^7+vM0R$8b3k1aSU&kxvVPfOz3~)0O2iTYinV9_5{pF18j4b{o`=@AZIOAwwedB2@ ztXI1F04mg{<>a-gdFoRjq$6#FaevDn$^06L)k%wYq03&ysdXE+LL1#w$rRS1Y;BoS zH1x}{ms>LHWmdtP(ydD!aRdAa(d@csEo z0EF9L>%tppp`CZ2)jVb8AuoYyu;d^wfje6^n6`A?6$&%$p>HcE_De-Zh)%3o5)LDa zskQ}%o7?bg$xUj|n8gN9YB)z!N&-K&!_hVQ?#SFj+MpQA4@4oq!UQ$Vm3B`W_Pq3J z=ngFP4h_y=`Iar<`EESF9){%YZVyJqLPGq07TP7&fSDmnYs2NZQKiR%>){imTBJth zPHr@p>8b+N@~%43rSeNuOz;rgEm?14hNtI|KC6Xz1d?|2J`QS#`OW7gTF_;TPPxu@ z)9J9>3Lx*bc>Ielg|F3cou$O0+<b34_*ZJhpS&$8DP>s%47a)4ZLw`|>s=P_J4u z?I_%AvR_z8of@UYWJV?~c4Yb|A!9n!LEUE6{sn@9+D=0w_-`szJ_T++x3MN$v-)0d zy`?1QG}C^KiNlnJBRZBLr4G~15V3$QqC%1G5b#CEB0VTr#z?Ug%Jyv@a`QqAYUV~^ zw)d|%0g&kl{j#FMdf$cn(~L@8s~6eQ)6{`ik(RI(o9s0g30Li{4YoxcVoYd+LpeLz zai?~r)UcbYr@lv*Z>E%BsvTNd`Sc?}*}>mzJ|cr0Y(6rA7H_6&t>F{{mJ^xovc2a@ zFGGDUcGgI-z6H#o@Gj29C=Uy{wv zQHY2`HZu8+sBQK*_~I-_>fOTKEAQ8_Q~YE$c?cSCxI;vs-JGO`RS464Ft06rpjn+a zqRS0Y3oN(9HCP@{J4mOWqIyD8PirA!pgU^Ne{LHBG;S*bZpx3|JyQDGO&(;Im8!ed zNdpE&?3U?E@O~>`@B;oY>#?gXEDl3pE@J30R1;?QNNxZ?YePc)3=NS>!STCrXu*lM z69WkLB_RBwb1^-zEm*tkcHz3H;?v z;q+x0Jg$|?5;e1-kbJnuT+^$bWnYc~1qnyVTKh*cvM+8yJT-HBs1X@cD;L$su65;i z2c1MxyL~NuZ9+)hF=^-#;dS#lFy^Idcb>AEDXu1!G4Kd8YPy~0lZz$2gbv?su}Zn} zGtIbeYz3X8OA9{sT(aleold_?UEV{hWRl(@)NH6GFH@$<8hUt=dNte%e#Jc>7u9xi zuqv!CRE@!fmZZ}3&@$D>p0z=*dfQ_=IE4bG0hLmT@OP>x$e`qaqf_=#baJ8XPtOpWi%$ep1Y)o2(sR=v)M zt(z*pGS$Z#j_xq_lnCr+x9fwiT?h{NEn#iK(o)G&Xw-#DK?=Ms6T;%&EE${Gq_%99 z6(;P~jPKq9llc+cmI(MKQ6*7PcL)BmoI}MYFO)b3-{j>9FhNdXLR<^mnMP`I7z0v` zj3wxcXAqi4Z0kpeSf>?V_+D}NULgU$DBvZ^=0G8Bypd7P2>;u`yW9`%4~&tzNJpgp zqB+iLIM~IkB;ts!)exn643mAJ8-WlgFE%Rpq!UMYtB?$5QAMm)%PT0$$2{>Yu7&U@ zh}gD^Qdgu){y3ANdB5{75P;lRxSJPSpQPMJOiwmpMdT|?=q;&$aTt|dl~kvS z+*i;6cEQJ1V`R4Fd>-Uzsc=DPQ7A7#VPCIf!R!KK%LM&G%MoZ0{-8&99H!|UW$Ejv zhDLX3ESS6CgWTm#1ZeS2HJb`=UM^gsQ84dQpX(ESWSkjn>O zVxg%`@mh(X9&&wN$lDIc*@>rf?C0AD_mge3f2KkT6kGySOhXqZjtA?5z`vKl_{(5g z&%Y~9p?_DL{+q@siT~*3Q*$nWXQfNN;%s_eHP_A;O`N`SaoB z6xYR;z_;HQ2xAa9xKgx~2f2xEKiEDpGPH1d@||v#f#_Ty6_gY>^oZ#xac?pc-F`@ z*}8sPV@xiz?efDMcmmezYVw~qw=vT;G1xh+xRVBkmN66!u(mRG3G6P#v|;w@anEh7 zCf94arw%YB*=&3=RTqX?z4mID$W*^+&d6qI*LA-yGme;F9+wTsNXNaX~zl2+qIK&D-aeN4lr0+yP;W>|Dh?ms_ogT{DT+ ztXFy*R7j4IX;w@@R9Oct5k2M%&j=c_rWvoul+` z<18FH5D@i$P38W9VU2(EnEvlJ(SHCqTNBa)brkIjGP|jCnK&Qi%97tikU}Y#3L?s! z2ujL%YiHO-#!|g5066V01hgT#>fzls7P>+%D~ogOT&!Whb4iF=CnCto82Yb#b`YoVsj zS2q^W0Rj!RrM@=_GuPQy5*_X@Zmu`TKSbqEOP@;Ga&Rrr>#H@L41@ZX)LAkbo{G8+ z;!5EH6vv-ip0`tLB)xUuOX(*YEDSWf?PIxXe`+_B8=KH#HFCfthu}QJylPMTNmoV; zC63g%?57(&osaH^sxCyI-+gwVB|Xs2TOf=mgUAq?V~N_5!4A=b{AXbDae+yABuuu3B_XSa4~c z1s-OW>!cIkjwJf4ZhvT|*IKaRTU)WAK=G|H#B5#NB9<{*kt?7`+G*-^<)7$Iup@Um z7u*ABkG3F*Foj)W9-I&@BrN8(#$7Hdi`BU#SR1Uz4rh&=Ey!b76Qo?RqBJ!U+rh(1 znw@xw5$)4D8OWtB_^pJO*d~2Mb-f~>I!U#*=Eh*xa6$LX?4Evp4%;ENQR!mF4`f7F zpG!NX=qnCwE8@NAbQV`*?!v0;NJ(| zBip8}VgFVsXFqslXUV>_Z>1gmD(7p#=WACXaB|Y`=Kxa=p@_ALsL&yAJ`*QW^`2@% zW7~Yp(Q@ihmkf{vMF?kqkY%SwG^t&CtfRWZ{syK@W$#DzegcQ1>~r7foTw3^V1)f2Tq_5f$igmfch;8 zT-<)?RKcCdQh6x^mMEOS;4IpQ@F2q-4IC4%*dU@jfHR4UdG>Usw4;7ESpORL|2^#jd+@zxz{(|RV*1WKrw-)ln*8LnxVkKDfGDHA%7`HaiuvhMu%*mY9*Ya{Ti#{DW?i0 zXXsp+Bb(_~wv(3t70QU3a$*<$1&zm1t++x#wDLCRI4K)kU?Vm9n2c0m@TyUV&&l9%}fulj!Z9)&@yIcQ3gX}l0b1LbIh4S z5C*IDrYxR%qm4LVzSk{0;*npO_SocYWbkAjA6(^IAwUnoAzw_Uo}xYFo?Y<-4Zqec z&k7HtVlFGyt_pA&kX%P8PaRD8y!Wsnv}NMLNLy-CHZf(ObmzV|t-iC#@Z9*d-zUsx zxcYWw{H)nYXVdnJu5o-U+fn~W z-$h1ax>h{NlWLA7;;6TcQHA>UJB$KNk74T1xNWh9)kwK~wX0m|Jo_Z;g;>^E4-k4R zRj#pQb-Hg&dAh}*=2;JY*aiNZzT=IU&v|lQY%Q|=^V5pvTR7^t9+@+ST&sr!J1Y9a z514dYZn5rg6@4Cy6P`-?!3Y& z?B*5zw!mTiD2)>f@3XYrW^9V-@%YFkE_;PCyCJ7*?_3cR%tHng9%ZpIU}LJM=a+0s z(SDDLvcVa~b9O!cVL8)Q{d^R^(bbG=Ia$)dVN_tGMee3PMssZ7Z;c^Vg_1CjZYTnq z)wnF8?=-MmqVOMX!iE?YDvHCN?%TQtKJMFHp$~kX4}jZ;EDqP$?jqJZjoa2PM@$uZ zF4}iab1b5ep)L;jdegC3{K4VnCH#OV;pRcSa(&Nm50ze-yZ8*cGv;@+N+A?ncc^2z9~|(xFhwOHmPW@ zR5&)E^YKQj@`g=;zJ_+CLamsPuvppUr$G1#9urUj+p-mPW_QSSHkPMS!52t>Hqy|g z_@Yu3z%|wE=uYq8G>4`Q!4zivS}+}{m5Zjr7kMRGn_p&hNf|pc&f9iQ`^%78rl#~8 z;os@rpMA{ZioY~(Rm!Wf#Wx##A0PthOI341QiJ=G*#}pDAkDm+{0kz&*NB?rC0-)glB{0_Tq*^o zVS1>3REsv*Qb;qg!G^9;VoK)P*?f<*H&4Su1=}bP^Y<2PwFpoqw#up4IgX3L z`w~8jsFCI3k~Y9g(Y9Km`y$0FS5vHb)kb)Jb6q-9MbO{Hbb zxg?IWQ1ZIGgE}wKm{axO6CCh~4DyoFU+i1xn#oyfe+<{>=^B5tm!!*1M?AW8c=6g+%2Ft97_Hq&ZmOGvqGQ!Bn<_Vw`0DRuDoB6q8ME<;oL4kocr8E$NGoLI zXWmI7Af-DR|KJw!vKp2SI4W*x%A%5BgDu%8%Iato+pWo5`vH@!XqC!yK}KLzvfS(q z{!y(S-PKbk!qHsgVyxKsQWk_8HUSSmslUA9nWOjkKn0%cwn%yxnkfxn?Y2rysXKS=t-TeI%DN$sQ{lcD!(s>(4y#CSxZ4R} zFDI^HPC_l?uh_)-^ppeYRkPTPu~V^0Mt}#jrTL1Q(M;qVt4zb(L|J~sxx7Lva9`mh zz!#A9tA*6?q)xThc7(gB2Ryam$YG4qlh00c}r&$y6u zIN#Qxn{7RKJ+_r|1G1KEv!&uKfXpOVZ8tK{M775ws%nDyoZ?bi3NufNbZs)zqXiqc zqOsK@^OnlFMAT&mO3`@3nZP$3lLF;ds|;Z{W(Q-STa2>;)tjhR17OD|G>Q#zJHb*> zMO<{WIgB%_4MG0SQi2;%f0J8l_FH)Lfaa>*GLobD#AeMttYh4Yfg22@q4|Itq};NB z8;o*+@APqy@fPgrc&PTbGEwdEK=(x5K!If@R$NiO^7{#j9{~w=RBG)ZkbOw@$7Nhl zyp{*&QoVBd5lo{iwl2gfyip@}IirZK;ia(&ozNl!-EEYc=QpYH_= zJkv7gA{!n4up6$CrzDJIBAdC7D5D<_VLH*;OYN>_Dx3AT`K4Wyx8Tm{I+xplKP6k7 z2sb!i7)~%R#J0$|hK?~=u~rnH7HCUpsQJujDDE*GD`qrWWog+C+E~GGy|Hp_t4--} zrxtrgnPh}r=9o}P6jpAQuDN}I*GI`8&%Lp-C0IOJt#op)}XSr!ova@w{jG2V=?GXl3zEJJFXg)U3N>BQP z*Lb@%Mx|Tu;|u>$-K(q^-HG!EQ3o93%w(A7@ngGU)HRWoO&&^}U$5x+T&#zri>6ct zXOB#EF-;z3j311K`jrYyv6pOPF=*`SOz!ack=DuEi({UnAkL5H)@R?YbRKAeP|06U z?-Ns0ZxD0h9D8)P66Sq$w-yF+1hEVTaul%&=kKDrQtF<$RnQPZ)ezm1`aHIjAY=!S z`%vboP`?7mItgEo4w50C*}Ycqp9_3ZEr^F1;cEhkb`BNhbc6PvnXu@wi=AoezF4~K zkxx%ps<8zb=wJ+9I8o#do)&{(=yAlNdduaDn!=xGSiuo~fLw~Edw$6;l-qaq#Z7?# zGrdU(Cf-V@$x>O%yRc6!C1Vf`b19ly;=mEu8u9|zitcG^O`lbNh}k=$%a)UHhDwTEKis2yc4rBGR>l*(B$AC7ung&ssaZGkY-h(fpwcPyJSx*9EIJMRKbMP9}$nVrh6$g-Q^5Cw)BeWqb-qi#37ZXKL!GR;ql)~ z@PP*-oP?T|ThqlGKR84zi^CN z4TZ1A)7vL>ivoL2EU_~xl-P{p+sE}9CRwGJDKy{>0KP+gj`H9C+4fUMPnIB1_D`A- z$1`G}g0lQmqMN{Y&8R*$xYUB*V}dQPxGVZQ+rH!DVohIoTbh%#z#Tru%Px@C<=|og zGDDwGq7yz`%^?r~6t&>x*^We^tZ4!E4dhwsht#Pb1kCY{q#Kv;z%Dp#Dq;$vH$-(9 z8S5tutZ}&JM2Iw&Y-7KY4h5BBvS=Ove0#+H2qPdR)WyI zYcj)vB=MA{7T|3Ij_PN@FM@w(C9ANBq&|NoW30ccr~i#)EcH)T^3St~rJ0HKKd4wr z@_+132;Bj+>UC@h)Ap*8B4r5A1lZ!Dh%H7&&hBnlFj@eayk=VD*i5AQc z$uN8YG#PL;cuQa)Hyt-}R?&NAE1QT>svJDKt*)AQOZAJ@ zyxJoBebiobHeFlcLwu_iI&NEZuipnOR;Tn;PbT1Mt-#5v5b*8ULo7m)L-eti=UcGf zRZXidmxeFgY!y80-*PH-*=(-W+fK%KyUKpg$X@tuv``tXj^*4qq@UkW$ZrAo%+hay zU@a?z&2_@y)o@D!_g>NVxFBO!EyB&6Z!nd4=KyDP^hl!*(k{dEF6@NkXztO7gIh zQ&PC+p-8WBv;N(rpfKdF^@Z~|E6pa)M1NBUrCZvLRW$%N%xIbv^uv?=C!=dDVq3%* zgvbEBnG*JB*@vXx8>)7XL*!{1Jh=#2UrByF7U?Rj_}VYw88BwqefT_cCTv8aTrRVjnn z1HNCF=44?*&gs2`vCGJVHX@kO z240eo#z+FhI0=yy6NHQwZs}a+J~4U-6X`@ zZ7j+tb##m`x%J66$a9qXDHG&^kp|GkFFMmjD(Y-k_ClY~N$H|n@NkSDz=gg?*2ga5 z)+f)MEY>2Lp15;~o`t`qj;S>BaE;%dv@Ux11yq}I(k|o&`5UZFUHn}1kE^gIK@qV& z!S2IhyU;->VfA4Qb}m7YnkIa9%z{l~iPWo2YPk-`hy2-Eg=6E$21plQA5W2qMZDFU z-a-@Dndf%#on6chT`dOKnU9}BJo|kJwgGC<^nfo34zOKH96LbWY7@Wc%EoFF=}`VU zksP@wd%@W;-p!e^&-)N7#oR331Q)@9cx=mOoU?_Kih2!Le*8fhsZ8Qvo6t2vt+UOZ zw|mCB*t2%z21YqL>whu!j?s~}-L`OS+jdg1(XnmYw$rg~r(?5Y+qTg`$F}q3J?GtL z@BN&8#`u2RqkdG4yGGTus@7U_%{6C{XAhFE!2SelH?KtMtX@B1GBhEIDL-Bj#~{4! zd}p7!#XE9Lt;sy@p5#Wj*jf8zGv6tTotCR2X$EVOOup;GnRPRVU5A6N@Lh8?eA7k? zn~hz&gY;B0ybSpF?qwQ|sv_yO=8}zeg2$0n3A8KpE@q26)?707pPw?H76lCpjp=5r z6jjp|auXJDnW}uLb6d7rsxekbET9(=zdTqC8(F5@NNqII2+~yB;X5iJNQSiv`#ozm zf&p!;>8xAlwoxUC3DQ#!31ylK%VrcwS<$WeCY4V63V!|221oj+5#r}fGFQ}|uwC0) zNl8(CF}PD`&Sj+p{d!B&&JtC+VuH z#>US`)YQrhb6lIAYb08H22y(?)&L8MIQsA{26X`R5Km{YU)s!x(&gIsjDvq63@X`{ z=7{SiH*_ZsPME#t2m|bS76Uz*z{cpp1m|s}HIX}Ntx#v7Eo!1%G9__4dGSGl`p+xi zZ!VK#Qe;Re=9bqXuW+0DSP{uZ5-QXrNn-7qW19K0qU}OhVru7}3vqsG?#D67 zb}crN;QwsH*vymw(maZr_o|w&@sQki(X+D)gc5Bt&@iXisFG;eH@5d43~Wxq|HO(@ zV-rip4n#PEkHCWCa5d?@cQp^B;I-PzOfag|t-cuvTapQ@MWLmh*41NH`<+A+JGyKX zyYL6Ba7qqa5j@3lOk~`OMO7f0!@FaOeZxkbG@vXP(t3#U*fq8=GAPqUAS>vW2uxMk{a(<0=IxB;# zMW;M+owrHaZBp`3{e@7gJCHP!I(EeyGFF;pdFPdeP+KphrulPSVidmg#!@W`GpD&d z9p6R`dpjaR2E1Eg)Ws{BVCBU9-aCgN57N~uLvQZH`@T+2eOBD%73rr&sV~m#2~IZx zY_8f8O;XLu2~E3JDXnGhFvsyb^>*!D>5EtlKPe%kOLv6*@=Jpci`8h0z?+fbBUg_7 zu6DjqO=$SjAv{|Om5)nz41ZkS4E_|fk%NDY509VV5yNeo%O|sb>7C#wj8mL9cEOFh z>nDz%?vb!h*!0dHdnxDA>97~EoT~!N40>+)G2CeYdOvJr5^VnkGz)et&T9hrD(VAgCAJjQ7V$O?csICB*HFd^k@$M5*v$PZJD-OVL?Ze(U=XGqZPVG8JQ z<~ukO%&%nNXYaaRibq#B1KfW4+XMliC*Tng2G(T1VvP;2K~;b$EAqthc${gjn_P!b zs62UT(->A>!ot}cJXMZHuy)^qfqW~xO-In2);e>Ta{LD6VG2u&UT&a@>r-;4<)cJ9 zjpQThb4^CY)Ev0KR7TBuT#-v}W?Xzj{c7$S5_zJA57Qf=$4^npEjl9clH0=jWO8sX z3Fuu0@S!WY>0XX7arjH`?)I<%2|8HfL!~#c+&!ZVmhbh`wbzy0Ux|Jpy9A{_7GGB0 zadZ48dW0oUwUAHl%|E-Q{gA{z6TXsvU#Hj09<7i)d}wa+Iya)S$CVwG{4LqtB>w%S zKZx(QbV7J9pYt`W4+0~f{hoo5ZG<0O&&5L57oF%hc0xGJ@Zrg_D&lNO=-I^0y#3mxCSZFxN2-tN_mU@7<@PnWG?L5OSqkm8TR!`| zRcTeWH~0z1JY^%!N<(TtxSP5^G9*Vw1wub`tC-F`=U)&sJVfvmh#Pi`*44kSdG};1 zJbHOmy4Ot|%_?@$N?RA9fF?|CywR8Sf(SCN_luM8>(u0NSEbKUy7C(Sk&OuWffj)f za`+mo+kM_8OLuCUiA*CNE|?jra$M=$F3t+h-)?pXz&r^F!ck;r##`)i)t?AWq-9A9 zSY{m~TC1w>HdEaiR*%j)L);H{IULw)uxDO>#+WcBUe^HU)~L|9#0D<*Ld459xTyew zbh5vCg$a>`RCVk)#~ByCv@Ce!nm<#EW|9j><#jQ8JfTmK#~jJ&o0Fs9jz0Ux{svdM4__<1 zrb>H(qBO;v(pXPf5_?XDq!*3KW^4>(XTo=6O2MJdM^N4IIcYn1sZZpnmMAEdt}4SU zPO54j2d|(xJtQ9EX-YrlXU1}6*h{zjn`in-N!Ls}IJsG@X&lfycsoCemt_Ym(PXhv zc*QTnkNIV=Ia%tg%pwJtT^+`v8ng>;2~ps~wdqZSNI7+}-3r+#r6p`8*G;~bVFzg= z!S3&y)#iNSUF6z;%o)%h!ORhE?CUs%g(k2a-d576uOP2@QwG-6LT*G!I$JQLpd`cz z-2=Brr_+z96a0*aIhY2%0(Sz=|D`_v_7h%Yqbw2)8@1DwH4s*A82krEk{ zoa`LbCdS)R?egRWNeHV8KJG0Ypy!#}kslun?67}^+J&02!D??lN~t@;h?GS8#WX`)6yC**~5YNhN_Hj}YG<%2ao^bpD8RpgV|V|GQwlL27B zEuah|)%m1s8C6>FLY0DFe9Ob66fo&b8%iUN=y_Qj;t3WGlNqP9^d#75ftCPA*R4E8 z)SWKBKkEzTr4JqRMEs`)0;x8C35yRAV++n(Cm5++?WB@ya=l8pFL`N0ag`lWhrYo3 zJJ$< zQ*_YAqIGR*;`VzAEx1Pd4b3_oWtdcs7LU2#1#Ls>Ynvd8k^M{Ef?8`RxA3!Th-?ui{_WJvhzY4FiPxA?E4+NFmaC-Uh*a zeLKkkECqy>Qx&1xxEhh8SzMML=8VP}?b*sgT9ypBLF)Zh#w&JzP>ymrM?nnvt!@$2 zh>N$Q>mbPAC2kNd&ab;FkBJ}39s*TYY0=@e?N7GX>wqaM>P=Y12lciUmve_jMF0lY zBfI3U2{33vWo(DiSOc}!5##TDr|dgX1Uojq9!vW3$m#zM_83EGsP6&O`@v-PDdO3P z>#!BEbqpOXd5s?QNnN!p+92SHy{sdpePXHL{d@c6UilT<#~I!tH$S(~o}c#(j<2%! zQvm}MvAj-95Ekx3D4+|e%!?lO(F+DFw9bxb-}rsWQl)b44###eUg4N?N-P(sFH2hF z`{zu?LmAxn2=2wCE8?;%ZDi#Y;Fzp+RnY8fWlzVz_*PDO6?Je&aEmuS>=uCXgdP6r zoc_JB^TA~rU5*geh{G*gl%_HnISMS~^@{@KVC;(aL^ZA-De+1zwUSXgT>OY)W?d6~ z72znET0m`53q%AVUcGraYxIcAB?OZA8AT!uK8jU+=t;WneL~|IeQ>$*dWa#x%rB(+ z5?xEkZ&b{HsZ4Ju9TQ|)c_SIp`7r2qMJgaglfSBHhl)QO1aNtkGr0LUn{@mvAt=}nd7#>7ru}&I)FNsa*x?Oe3-4G`HcaR zJ}c%iKlwh`x)yX1vBB;-Nr=7>$~(u=AuPX2#&Eh~IeFw%afU+U)td0KC!pHd zyn+X$L|(H3uNit-bpn7%G%{&LsAaEfEsD?yM<;U2}WtD4KuVKuX=ec9X zIe*ibp1?$gPL7<0uj*vmj2lWKe`U(f9E{KVbr&q*RsO;O>K{i-7W)8KG5~~uS++56 zm@XGrX@x+lGEjDQJp~XCkEyJG5Y57omJhGN{^2z5lj-()PVR&wWnDk2M?n_TYR(gM zw4kQ|+i}3z6YZq8gVUN}KiYre^sL{ynS}o{z$s&I z{(rWaLXxcQ=MB(Cz7W$??Tn*$1y(7XX)tv;I-{7F$fPB%6YC7>-Dk#=Y8o1=&|>t5 zV_VVts>Eb@)&4%m}!K*WfLoLl|3FW)V~E1Z!yu`Sn+bAP5sRDyu7NEbLt?khAyz-ZyL-}MYb&nQ zU16f@q7E1rh!)d%f^tTHE3cVoa%Xs%rKFc|temN1sa)aSlT*)*4k?Z>b3NP(IRXfq zlB^#G6BDA1%t9^Nw1BD>lBV(0XW5c?l%vyB3)q*;Z5V~SU;HkN;1kA3Nx!$!9wti= zB8>n`gt;VlBt%5xmDxjfl0>`K$fTU-C6_Z;!A_liu0@Os5reMLNk;jrlVF^FbLETI zW+Z_5m|ozNBn7AaQ<&7zk}(jmEdCsPgmo%^GXo>YYt82n&7I-uQ%A;k{nS~VYGDTn zlr3}HbWQG6xu8+bFu^9%%^PYCbkLf=*J|hr>Sw+#l(Y#ZGKDufa#f-f0k-{-XOb4i zwVG1Oa0L2+&(u$S7TvedS<1m45*>a~5tuOZ;3x%!f``{=2QQlJk|b4>NpD4&L+xI+ z+}S(m3}|8|Vv(KYAGyZK5x*sgwOOJklN0jsq|BomM>OuRDVFf_?cMq%B*iQ*&|vS9 zVH7Kh)SjrCBv+FYAE=$0V&NIW=xP>d-s7@wM*sdfjVx6-Y@=~>rz%2L*rKp|*WXIz z*vR^4tV&7MQpS9%{9b*>E9d_ls|toL7J|;srnW{l-}1gP_Qr-bBHt=}PL@WlE|&KH zCUmDLZb%J$ZzNii-5VeygOM?K8e$EcK=z-hIk63o4y63^_*RdaitO^THC{boKstphXZ2Z+&3ToeLQUG(0Frs?b zCxB+65h7R$+LsbmL51Kc)pz_`YpGEzFEclzb=?FJ=>rJwgcp0QH-UuKRS1*yCHsO) z-8t?Zw|6t($Eh&4K+u$I7HqVJBOOFCRcmMMH};RX_b?;rnk`rz@vxT_&|6V@q0~Uk z9ax|!pA@Lwn8h7syrEtDluZ6G!;@=GL> zse#PRQrdDs=qa_v@{Wv(3YjYD0|qocDC;-F~&{oaTP?@pi$n z1L6SlmFU2~%)M^$@C(^cD!y)-2SeHo3t?u3JiN7UBa7E2 z;<+_A$V084@>&u)*C<4h7jw9joHuSpVsy8GZVT;(>lZ(RAr!;)bwM~o__Gm~exd`K zKEgh2)w?ReH&syI`~;Uo4`x4$&X+dYKI{e`dS~bQuS|p zA`P_{QLV3r$*~lb=9vR^H0AxK9_+dmHX}Y} zIV*#65%jRWem5Z($ji{!6ug$En4O*=^CiG=K zp4S?+xE|6!cn$A%XutqNEgUqYY3fw&N(Z6=@W6*bxdp~i_yz5VcgSj=lf-6X1Nz75 z^DabwZ4*70$$8NsEy@U^W67tcy7^lNbu;|kOLcJ40A%J#pZe0d#n zC{)}+p+?8*ftUlxJE*!%$`h~|KZSaCb=jpK3byAcuHk7wk@?YxkT1!|r({P*KY^`u z!hw#`5$JJZGt@nkBK_nwWA31_Q9UGvv9r-{NU<&7HHMQsq=sn@O?e~fwl20tnSBG* zO%4?Ew6`aX=I5lqmy&OkmtU}bH-+zvJ_CFy z_nw#!8Rap5Wcex#5}Ldtqhr_Z$}@jPuYljTosS1+WG+TxZ>dGeT)?ZP3#3>sf#KOG z0)s%{cEHBkS)019}-1A2kd*it>y65-C zh7J9zogM74?PU)0c0YavY7g~%j%yiWEGDb+;Ew5g5Gq@MpVFFBNOpu0x)>Yn>G6uo zKE%z1EhkG_N5$a8f6SRm(25iH#FMeaJ1^TBcBy<04ID47(1(D)q}g=_6#^V@yI?Y&@HUf z`;ojGDdsvRCoTmasXndENqfWkOw=#cV-9*QClpI03)FWcx(m5(P1DW+2-{Hr-`5M{v##Zu-i-9Cvt;V|n)1pR^y ztp3IXzHjYWqabuPqnCY9^^;adc!a%Z35VN~TzwAxq{NU&Kp35m?fw_^D{wzB}4FVXX5Zk@#={6jRh%wx|!eu@Xp;%x+{2;}!&J4X*_SvtkqE#KDIPPn@ z5BE$3uRlb>N<2A$g_cuRQM1T#5ra9u2x9pQuqF1l2#N{Q!jVJ<>HlLeVW|fN|#vqSnRr<0 zTVs=)7d`=EsJXkZLJgv~9JB&ay16xDG6v(J2eZy;U%a@EbAB-=C?PpA9@}?_Yfb&) zBpsih5m1U9Px<+2$TBJ@7s9HW>W){i&XKLZ_{1Wzh-o!l5_S+f$j^RNYo85}uVhN# zq}_mN-d=n{>fZD2Lx$Twd2)}X2ceasu91}n&BS+4U9=Y{aZCgV5# z?z_Hq-knIbgIpnkGzJz-NW*=p?3l(}y3(aPCW=A({g9CpjJfYuZ%#Tz81Y)al?!S~ z9AS5#&nzm*NF?2tCR#|D-EjBWifFR=da6hW^PHTl&km-WI9*F4o>5J{LBSieVk`KO z2(^9R(zC$@g|i3}`mK-qFZ33PD34jd_qOAFj29687wCUy>;(Hwo%Me&c=~)V$ua)V zsaM(aThQ3{TiM~;gTckp)LFvN?%TlO-;$y+YX4i`SU0hbm<})t0zZ!t1=wY&j#N>q zONEHIB^RW6D5N*cq6^+?T}$3m|L{Fe+L!rxJ=KRjlJS~|z-&CC{#CU8`}2|lo~)<| zk?Wi1;Cr;`?02-C_3^gD{|Ryhw!8i?yx5i0v5?p)9wZxSkwn z3C;pz25KR&7{|rc4H)V~y8%+6lX&KN&=^$Wqu+}}n{Y~K4XpI-#O?L=(2qncYNePX zTsB6_3`7q&e0K67=Kg7G=j#?r!j0S^w7;0?CJbB3_C4_8X*Q%F1%cmB{g%XE&|IA7 z(#?AeG{l)s_orNJp!$Q~qGrj*YnuKlV`nVdg4vkTNS~w$4d^Oc3(dxi(W5jq0e>x} z(GN1?u2%Sy;GA|B%Sk)ukr#v*UJU%(BE9X54!&KL9A^&rR%v zIdYt0&D59ggM}CKWyxGS@ z>T#})2Bk8sZMGJYFJtc>D#k0+Rrrs)2DG;(u(DB_v-sVg=GFMlSCx<&RL;BH}d6AG3VqP!JpC0Gv6f8d|+7YRC@g|=N=C2 zo>^0CE0*RW?W))S(N)}NKA)aSwsR{1*rs$(cZIs?nF9)G*bSr%%SZo^YQ|TSz={jX z4Z+(~v_>RH0(|IZ-_D_h@~p_i%k^XEi+CJVC~B zsPir zA0Jm2yIdo4`&I`hd%$Bv=Rq#-#bh{Mxb_{PN%trcf(#J3S1UKDfC1QjH2E;>wUf5= ze8tY9QSYx0J;$JUR-0ar6fuiQTCQP#P|WEq;Ez|*@d?JHu-(?*tTpGHC+=Q%H>&I> z*jC7%nJIy+HeoURWN%3X47UUusY2h7nckRxh8-)J61Zvn@j-uPA@99|y48pO)0XcW zX^d&kW^p7xsvdX?2QZ8cEUbMZ7`&n{%Bo*xgFr4&fd#tHOEboQos~xm8q&W;fqrj} z%KYnnE%R`=`+?lu-O+J9r@+$%YnqYq!SVs>xp;%Q8p^$wA~oynhnvIFp^)Z2CvcyC zIN-_3EUHW}1^VQ0;Oj>q?mkPx$Wj-i7QoXgQ!HyRh6Gj8p~gH22k&nmEqUR^)9qni{%uNeV{&0-H60C zibHZtbV=8=aX!xFvkO}T@lJ_4&ki$d+0ns3FXb+iP-VAVN`B7f-hO)jyh#4#_$XG%Txk6M<+q6D~ zi*UcgRBOoP$7P6RmaPZ2%MG}CMfs=>*~(b97V4+2qdwvwA@>U3QQAA$hiN9zi%Mq{ z*#fH57zUmi)GEefh7@`Uy7?@@=BL7cXbd{O9)*lJh*v!@ z-6}p9u0AreiGauxn7JBEa-2w&d=!*TLJ49`U@D7%2ppIh)ynMaAE2Q4dl@47cNu{9 z&3vT#pG$#%hrXzXsj=&Ss*0;W`Jo^mcy4*L8b^sSi;H{*`zW9xX2HAtQ*sO|x$c6UbRA(7*9=;D~(%wfo(Z6#s$S zuFk`dr%DfVX5KC|Af8@AIr8@OAVj=6iX!~8D_P>p7>s!Hj+X0_t}Y*T4L5V->A@Zx zcm1wN;TNq=h`5W&>z5cNA99U1lY6+!!u$ib|41VMcJk8`+kP{PEOUvc@2@fW(bh5pp6>C3T55@XlpsAd#vn~__3H;Dz2w=t9v&{v*)1m4)vX;4 zX4YAjM66?Z7kD@XX{e`f1t_ZvYyi*puSNhVPq%jeyBteaOHo7vOr8!qqp7wV;)%jtD5>}-a?xavZ;i|2P3~7c)vP2O#Fb`Y&Kce zQNr7%fr4#S)OOV-1piOf7NgQvR{lcvZ*SNbLMq(olrdDC6su;ubp5un!&oT=jVTC3uTw7|r;@&y*s)a<{J zkzG(PApmMCpMmuh6GkM_`AsBE@t~)EDcq1AJ~N@7bqyW_i!mtHGnVgBA`Dxi^P93i z5R;}AQ60wy=Q2GUnSwz+W6C^}qn`S-lY7=J(3#BlOK%pCl=|RVWhC|IDj1E#+|M{TV0vE;vMZLy7KpD1$Yk zi0!9%qy8>CyrcRK`juQ)I};r)5|_<<9x)32b3DT1M`>v^ld!yabX6@ihf`3ZVTgME zfy(l-ocFuZ(L&OM4=1N#Mrrm_<>1DZpoWTO70U8+x4r3BpqH6z@(4~sqv!A9_L}@7 z7o~;|?~s-b?ud&Wx6==9{4uTcS|0-p@dKi0y#tPm2`A!^o3fZ8Uidxq|uz2vxf;wr zM^%#9)h^R&T;}cxVI(XX7kKPEVb);AQO?cFT-ub=%lZPwxefymBk+!H!W(o(>I{jW z$h;xuNUr#^0ivvSB-YEbUqe$GLSGrU$B3q28&oA55l)ChKOrwiTyI~e*uN;^V@g-Dm4d|MK!ol8hoaSB%iOQ#i_@`EYK_9ZEjFZ8Ho7P^er z^2U6ZNQ{*hcEm?R-lK)pD_r(e=Jfe?5VkJ$2~Oq^7YjE^5(6a6Il--j@6dBHx2Ulq z!%hz{d-S~i9Eo~WvQYDt7O7*G9CP#nrKE#DtIEbe_uxptcCSmYZMqT2F}7Kw0AWWC zPjwo0IYZ6klc(h9uL|NY$;{SGm4R8Bt^^q{e#foMxfCSY^-c&IVPl|A_ru!ebwR#7 z3<4+nZL(mEsU}O9e`^XB4^*m)73hd04HH%6ok^!;4|JAENnEr~%s6W~8KWD)3MD*+ zRc46yo<}8|!|yW-+KulE86aB_T4pDgL$XyiRW(OOcnP4|2;v!m2fB7Hw-IkY#wYfF zP4w;k-RInWr4fbz=X$J;z2E8pvAuy9kLJUSl8_USi;rW`kZGF?*Ur%%(t$^{Rg!=v zg;h3@!Q$eTa7S0#APEDHLvK%RCn^o0u!xC1Y0Jg!Baht*a4mmKHy~88md{YmN#x) zBOAp_i-z2h#V~*oO-9k(BizR^l#Vm%uSa^~3337d;f=AhVp?heJ)nlZGm`}D(U^2w z#vC}o1g1h?RAV^90N|Jd@M00PoNUPyA?@HeX0P7`TKSA=*4s@R;Ulo4Ih{W^CD{c8 ze(ipN{CAXP(KHJ7UvpOc@9SUAS^wKo3h-}BDZu}-qjdNlVtp^Z{|CxKOEo?tB}-4; zEXyDzGbXttJ3V$lLo-D?HYwZm7vvwdRo}P#KVF>F|M&eJ44n*ZO~0)#0e0Vy&j00I z{%IrnUvKp70P?>~J^$^0Wo%>le>re2ZSvRfes@dC-*e=DD1-j%<$^~4^4>Id5w^Fr z{RWL>EbUCcyC%1980kOYqZAcgdz5cS8c^7%vvrc@CSPIx;X=RuodO2dxk17|am?HJ@d~Mp_l8H?T;5l0&WGFoTKM{eP!L-a0O8?w zgBPhY78tqf^+xv4#OK2I#0L-cSbEUWH2z+sDur85*!hjEhFfD!i0Eyr-RRLFEm5(n z-RV6Zf_qMxN5S6#8fr9vDL01PxzHr7wgOn%0Htmvk9*gP^Um=n^+7GLs#GmU&a#U^4jr)BkIubQO7oUG!4CneO2Ixa`e~+Jp9m{l6apL8SOqA^ zvrfEUPwnHQ8;yBt!&(hAwASmL?Axitiqvx%KZRRP?tj2521wyxN3ZD9buj4e;2y6U zw=TKh$4%tt(eh|y#*{flUJ5t4VyP*@3af`hyY^YU3LCE3Z|22iRK7M7E;1SZVHbXF zKVw!L?2bS|kl7rN4(*4h2qxyLjWG0vR@`M~QFPsf^KParmCX;Gh4OX6Uy9#4e_%oK zv1DRnfvd$pu(kUoV(MmAc09ckDiuqS$a%!AQ1Z>@DM#}-yAP$l`oV`BDYpkqpk(I|+qk!yoo$TwWr6dRzLy(c zi+qbVlYGz0XUq@;Fm3r~_p%by)S&SVWS+wS0rC9bk^3K^_@6N5|2rtF)wI>WJ=;Fz zn8$h<|Dr%kN|nciMwJAv;_%3XG9sDnO@i&pKVNEfziH_gxKy{l zo`2m4rnUT(qenuq9B0<#Iy(RPxP8R)=5~9wBku=%&EBoZ82x1GlV<>R=hIqf0PK!V zw?{z9e^B`bGyg2nH!^x}06oE%J_JLk)^QyHLipoCs2MWIqc>vaxsJj(=gg1ZSa=u{ zt}od#V;e7sA4S(V9^<^TZ#InyVBFT(V#$fvI7Q+pgsr_2X`N~8)IOZtX}e(Bn(;eF zsNj#qOF_bHl$nw5!ULY{lNx@93Fj}%R@lewUuJ*X*1$K`DNAFpE z7_lPE+!}uZ6c?+6NY1!QREg#iFy=Z!OEW}CXBd~wW|r_9%zkUPR0A3m+@Nk%4p>)F zXVut7$aOZ6`w}%+WV$te6-IX7g2yms@aLygaTlIv3=Jl#Nr}nN zp|vH-3L03#%-1-!mY`1z?+K1E>8K09G~JcxfS)%DZbteGQnQhaCGE2Y<{ut#(k-DL zh&5PLpi9x3$HM82dS!M?(Z zEsqW?dx-K_GMQu5K54pYJD=5+Rn&@bGjB?3$xgYl-|`FElp}?zP&RAd<522c$Rv6} zcM%rYClU%JB#GuS>FNb{P2q*oHy}UcQ-pZ2UlT~zXt5*k-ZalE(`p7<`0n7i(r2k{ zb84&^LA7+aW1Gx5!wK!xTbw0slM?6-i32CaOcLC2B>ZRI16d{&-$QBEu1fKF0dVU>GTP05x2>Tmdy`75Qx! z^IG;HB9V1-D5&&)zjJ&~G}VU1-x7EUlT3QgNT<&eIDUPYey$M|RD6%mVkoDe|;2`8Z+_{0&scCq>Mh3hj|E*|W3;y@{$qhu77D)QJ` znD9C1AHCKSAHQqdWBiP`-cAjq7`V%~JFES1=i-s5h6xVT<50kiAH_dn0KQB4t*=ua zz}F@mcKjhB;^7ka@WbSJFZRPeYI&JFkpJ-!B z!ju#!6IzJ;D@$Qhvz9IGY5!%TD&(db3<*sCpZ?U#1^9RWQ zs*O-)j!E85SMKtoZzE^8{w%E0R0b2lwwSJ%@E}Lou)iLmPQyO=eirG8h#o&E4~eew z;h><=|4m0$`ANTOixHQOGpksXlF0yy17E&JksB4_(vKR5s$Ve+i;gco2}^RRJI+~R zWJ82WGigLIUwP!uSELh3AAs9HmY-kz=_EL-w|9}noKE#(a;QBpEx9 z4BT-zY=6dJT>72Hkz=9J1E=}*MC;zzzUWb@x(Ho8cU_aRZ?fxse5_Ru2YOvcr?kg&pt@v;{ai7G--k$LQtoYj+Wjk+nnZty;XzANsrhoH#7=xVqfPIW(p zX5{YF+5=k4_LBnhLUZxX*O?29olfPS?u*ybhM_y z*XHUqM6OLB#lyTB`v<BZ&YRs$N)S@5Kn_b3;gjz6>fh@^j%y2-ya({>Hd@kv{CZZ2e)tva7gxLLp z`HoGW);eRtov~Ro5tetU2y72~ zQh>D`@dt@s^csdfN-*U&o*)i3c4oBufCa0e|BwT2y%Y~=U7A^ny}tx zHwA>Wm|!SCko~UN?hporyQHRUWl3djIc722EKbTIXQ6>>iC!x+cq^sUxVSj~u)dsY zW8QgfZlE*2Os%=K;_vy3wx{0u!2%A)qEG-$R^`($%AOfnA^LpkB_}Dd7AymC)zSQr z>C&N8V57)aeX8ap!|7vWaK6=-3~ko9meugAlBKYGOjc#36+KJwQKRNa_`W@7;a>ot zdRiJkz?+QgC$b}-Owzuaw3zBVLEugOp6UeMHAKo2$m4w zpw?i%Lft^UtuLI}wd4(-9Z^*lVoa}11~+0|Hs6zAgJ01`dEA&^>Ai=mr0nC%eBd_B zzgv2G_~1c1wr*q@QqVW*Wi1zn=}KCtSwLjwT>ndXE_Xa22HHL_xCDhkM( zhbw+j4uZM|r&3h=Z#YrxGo}GX`)AZyv@7#7+nd-D?BZV>thtc|3jt30j$9{aIw9)v zDY)*fsSLPQTNa&>UL^RWH(vpNXT7HBv@9=*=(Q?3#H*crA2>KYx7Ab?-(HU~a275)MBp~`P)hhzSsbj|d`aBe(L*(;zif{iFJu**ZR zkL-tPyh!#*r-JVQJq>5b0?cCy!uSKef+R=$s3iA7*k*_l&*e!$F zYwGI;=S^0)b`mP8&Ry@{R(dPfykD&?H)na^ihVS7KXkxb36TbGm%X1!QSmbV9^#>A z-%X>wljnTMU0#d;tpw?O1W@{X-k*>aOImeG z#N^x?ehaaQd}ReQykp>i;92q@%$a!y1PNyPYDIvMm& zyYVwn;+0({W@3h(r&i#FuCDE)AC(y&Vu>4?1@j0|CWnhHUx4|zL7cdaA32RSk?wl% zMK^n42@i5AU>f70(huWfOwaucbaToxj%+)7hnG^CjH|O`A}+GHZyQ-X57(WuiyRXV zPf>0N3GJ<2Myg!sE4XJY?Z7@K3ZgHy8f7CS5ton0Eq)Cp`iLROAglnsiEXpnI+S8; zZn>g2VqLxi^p8#F#Laf3<00AcT}Qh&kQnd^28u!9l1m^`lfh9+5$VNv=?(~Gl2wAl zx(w$Z2!_oESg_3Kk0hUsBJ<;OTPyL(?z6xj6LG5|Ic4II*P+_=ac7KRJZ`(k2R$L# zv|oWM@116K7r3^EL*j2ktjEEOY9c!IhnyqD&oy7+645^+@z5Y|;0+dyR2X6^%7GD* zXrbPqTO}O={ z4cGaI#DdpP;5u?lcNb($V`l>H7k7otl_jQFu1hh>=(?CTPN#IPO%O_rlVX}_Nq;L< z@YNiY>-W~&E@=EC5%o_z<^3YEw)i_c|NXxHF{=7U7Ev&C`c^0Z4-LGKXu*Hkk&Av= zG&RAv{cR7o4${k~f{F~J48Ks&o(D@j-PQ2`LL@I~b=ifx3q!p6`d>~Y!<-^mMk3)e zhi1;(YLU5KH}zzZNhl^`0HT(r`5FfmDEzxa zk&J7WQ|!v~TyDWdXQ)!AN_Y%xM*!jv^`s)A`|F%;eGg27KYsrCE2H}7*r)zvum6B{ z$k5Har9pv!dcG%f|3hE(#hFH+12RZPycVi?2y`-9I7JHryMn3 z9Y8?==_(vOAJ7PnT<0&85`_jMD0#ipta~Q3M!q5H1D@Nj-YXI$W%OQplM(GWZ5Lpq z-He6ul|3<;ZQsqs!{Y7x`FV@pOQc4|N;)qgtRe(Uf?|YqZv^$k8On7DJ5>f2%M=TV zw~x}9o=mh$JVF{v4H5Su1pq66+mhTG6?F>Do}x{V(TgFwuLfvNP^ijkrp5#s4UT!~ zEU7pr8aA)2z1zb|X9IpmJykQcqI#(rS|A4&=TtWu@g^;JCN`2kL}%+K!KlgC z>P)v+uCeI{1KZpewf>C=?N7%1e10Y3pQCZST1GT5fVyB1`q)JqCLXM zSN0qlreH1=%Zg-5`(dlfSHI&2?^SQdbEE&W4#%Eve2-EnX>NfboD<2l((>>34lE%) zS6PWibEvuBG7)KQo_`?KHSPk+2P;`}#xEs}0!;yPaTrR#j(2H|#-CbVnTt_?9aG`o z(4IPU*n>`cw2V~HM#O`Z^bv|cK|K};buJ|#{reT8R)f+P2<3$0YGh!lqx3&a_wi2Q zN^U|U$w4NP!Z>5|O)>$GjS5wqL3T8jTn%Vfg3_KnyUM{M`?bm)9oqZP&1w1)o=@+(5eUF@=P~ zk2B5AKxQ96n-6lyjh&xD!gHCzD$}OOdKQQk7LXS-fk2uy#h{ktqDo{o&>O!6%B|)` zg?|JgcH{P*5SoE3(}QyGc=@hqlB5w;bnmF#pL4iH`TSuft$dE5j^qP2S)?)@pjRQZ zBfo6g>c!|bN-Y|(Wah2o61Vd|OtXS?1`Fu&mFZ^yzUd4lgu7V|MRdGj3e#V`=mnk- zZ@LHn?@dDi=I^}R?}mZwduik!hC%=Hcl56u{Wrk1|1SxlgnzG&e7Vzh*wNM(6Y!~m z`cm8Ygc1$@z9u9=m5vs1(XXvH;q16fxyX4&e5dP-{!Kd555FD6G^sOXHyaCLka|8j zKKW^E>}>URx736WWNf?U6Dbd37Va3wQkiE;5F!quSnVKnmaIRl)b5rM_ICu4txs+w zj}nsd0I_VG^<%DMR8Zf}vh}kk;heOQTbl ziEoE;9@FBIfR7OO9y4Pwyz02OeA$n)mESpj zdd=xPwA`nO06uGGsXr4n>Cjot7m^~2X~V4yH&- zv2llS{|und45}Pm1-_W@)a-`vFBpD~>eVP(-rVHIIA|HD@%7>k8JPI-O*<7X{L*Ik zh^K`aEN!BteiRaY82FVo6<^8_22=aDIa8P&2A3V<(BQ;;x8Zs-1WuLRWjQvKv1rd2 zt%+fZ!L|ISVKT?$3iCK#7whp|1ivz1rV*R>yc5dS3kIKy_0`)n*%bfNyw%e7Uo}Mnnf>QwDgeH$X5eg_)!pI4EJjh6?kkG2oc6Af0py z(txE}$ukD|Zn=c+R`Oq;m~CSY{ebu9?!is}01sOK_mB?{lSY33E=!KkKtMeI*FO2b z%95awv9;Z|UDp3xm+aP*5I!R-_M2;GxeCRx3ATS0iF<_Do2Mi)Hk2 zjBF35VB>(oamIYjunu?g0O-?LuOvtfs5F(iiIicbu$HMPPF%F>pE@hIRjzT)>aa=m zwe;H9&+2|S!m74!E3xfO{l3E_ab`Q^tZ4yH9=~o2DUEtEMDqG=&D*8!>?2uao%w`&)THr z^>=L3HJquY>6)>dW4pCWbzrIB+>rdr{s}}cL_?#!sOPztRwPm1B=!jP7lQG|Iy6rP zVqZDNA;xaUx&xUt?Ox|;`9?oz`C0#}mc<1Urs#vTW4wd{1_r`eX=BeSV z_9WV*9mz>PH6b^z{VYQJ1nSTSqOFHE9u>cY)m`Q>=w1NzUShxcHsAxasnF2BG;NQ; zqL1tjLjImz_`q=|bAOr_i5_NEijqYZ^;d5y3ZFj6kCYakJh**N_wbfH;ICXq?-p#r z{{ljNDPSytOaG#7=yPmA&5gyYI%^7pLnMOw-RK}#*dk=@usL;|4US?{@K%7esmc&n z5$D*+l&C9)Bo@$d;Nwipd!68&+NnOj^<~vRcKLX>e03E|;to;$ndgR;9~&S-ly5gf z{rzj+j-g$;O|u?;wwxrEpD=8iFzUHQfl{B>bLHqH(9P zI59SS2PEBE;{zJUlcmf(T4DrcO?XRWR}?fekN<($1&AJTRDyW+D*2(Gyi?Qx-i}gy z&BpIO!NeVdLReO!YgdUfnT}7?5Z#~t5rMWqG+$N2n%5o#Np6ccNly}#IZQsW4?|NV zR9hrcyP(l#A+U4XcQvT;4{#i)dU>HK>aS!k1<3s2LyAhm2(!Nu%vRC9T`_yn9D+r} z1i&U~IcQ?4xhZYyH6WL-f%}qIhZkc&}n2N0PM| z6|XA9d-y;!`D{p;xu*gv7a|zaZ*MiQ)}zPzW4GB0mr)}N-DmB&hl1&x`2@sxN572_ zS)RdJyR%<7kW0v3Q_|57JKy&9tUdbqz}|hwn84}U*0r^jt6Ssrp+#1y=JBcZ+F`f(N?O0XL1OFGN`1-r?S<#t4*C9|y~e)!UYZ zRQ3M8m%~M)VriIvn~XzoP;5qeu(ZI>Y#r zAd)J)G9)*BeE%gmm&M@Olg3DI_zokjh9NvdGbT z+u4(Y&uC6tBBefIg~e=J#8i1Zxr>RT)#rGaB2C71usdsT=}mm`<#WY^6V{L*J6v&l z1^Tkr6-+^PA)yC;s1O^3Q!)Reb=fxs)P~I*?i&j{Vbb(Juc?La;cA5(H7#FKIj0Or zgV0BO{DUs`I9HgQ{-!g@5P^Vr|C4}~w6b=#`Zx0XcVSd?(04HUHwK(gJNafgQNB9Z zCi3TgNXAeJ+x|X|b@27$RxuYYuNSUBqo#uyiH6H(b~K*#!@g__4i%HP5wb<+Q7GSb zTZjJw96htUaGZ89$K_iBo4xEOJ#DT#KRu9ozu!GH0cqR>hP$nk=KXM%Y!(%vWQ#}s zy=O#BZ>xjUejMH^F39Bf0}>D}yiAh^toa-ts#gt6Mk9h1D<9_mGMBhLT0Ce2O3d_U znaTkBaxd-8XgwSp5)x-pqX5=+{cSuk6kyl@k|5DQ!5zLUVV%1X9vjY0gerbuG6nwZu5KDMdq(&UMLZ zy?jW#F6joUtVyz`Y?-#Yc0=i*htOFwQ3`hk$8oq35D}0m$FAOp#UFTV3|U3F>@N?d zeXLZCZjRC($%?dz(41e~)CN10qjh^1CdAcY(<=GMGk@`b1ptA&L*{L@_M{%Vd5b*x#b1(qh=7((<_l%ZUaHtmgq} zjchBdiis{Afxf@3CjPR09E*2#X(`W#-n`~6PcbaL_(^3tfDLk?Nb6CkW9v!v#&pWJ3iV-9hz zngp#Q`w`r~2wt&cQ9#S7z0CA^>Mzm7fpt72g<0y-KT{G~l-@L#edmjZQ}7{*$mLgSdJfS$Ge{hrD=mr;GD)uYq8}xS zT>(w_;}894Kb}(P5~FOpFIEjadhmxD(PsZbKwa-qxVa7Oc7~ebPKMeN(pCRzq8s@l z`|l^*X1eK1+Spz--WkSW_nK`Cs@JmkY4+p=U91nJoy{tSH;TzuIyS)Q_(S@;Iakua zpuDo5W54Mo;jY@Ly1dY)j|+M%$FJ0`C=FW#%UvOd&?p}0QqL20Xt!#pr8ujy6CA-2 zFz6Ex5H1i)c9&HUNwG{8K%FRK7HL$RJwvGakleLLo}tsb>t_nBCIuABNo$G--_j!gV&t8L^4N6wC|aLC)l&w04CD6Vc#h^(YH@Zs4nwUGkhc_-yt{dK zMZ<%$swLmUl8`E~RLihGt@J5v;r;vT&*Q!Cx zZ55-zpb;W7_Q{tf$mQvF61(K>kwTq0x{#Din||)B{+6O#ArLi)kiHWVC4`fOT&B(h zw&YV`J1|^FLx~9Q%r-SFhYl4PywI7sF2Q$>4o50~dfp5nn}XHv-_DM?RGs#+4gM;% znU>k=81G~f6u%^Z{bcX&sUv*h|L+|mNq=W43y@{~C zpL-TW3hYPs0^*OqS#KQwA^CGG_A-6#`_{1LBCD&*3nY0UHWJj1D|VP%oQlFxLllaA zVI@2^)HZ%E*=RbQcFOKIP7?+|_xVK+2oG(t_EGl2y;Ovox zZb^qVpe!4^reKvpIBFzx;Ji=PmrV>uu-Hb>`s?k?YZQ?>av45>i(w0V!|n?AP|v5H zm`e&Tgli#lqGEt?=(?~fy<(%#nDU`O@}Vjib6^rfE2xn;qgU6{u36j_+Km%v*2RLnGpsvS+THbZ>p(B zgb{QvqE?~50pkLP^0(`~K& zjT=2Pt2nSnwmnDFi2>;*C|OM1dY|CAZ5R|%SAuU|5KkjRM!LW_)LC*A zf{f>XaD+;rl6Y>Umr>M8y>lF+=nSxZX_-Z7lkTXyuZ(O6?UHw^q; z&$Zsm4U~}KLWz8>_{p*WQ!OgxT1JC&B&>|+LE3Z2mFNTUho<0u?@r^d=2 z-av!n8r#5M|F%l;=D=S1mGLjgFsiYAOODAR}#e^a8 zfVt$k=_o}kt3PTz?EpLkt54dY}kyd$rU zVqc9SN>0c z753j-gdN~UiW*FUDMOpYEkVzP)}{Ds*3_)ZBi)4v26MQr140|QRqhFoP=a|;C{#KS zD^9b-9HM11W+cb1Y)HAuk<^GUUo(ut!5kILBzAe)Vaxwu4Up!7Ql*#DDu z>EB84&xSrh>0jT!*X81jJQq$CRHqNj29!V3FN9DCx)~bvZbLwSlo3l^zPb1sqBnp) zfZpo|amY^H*I==3#8D%x3>zh#_SBf?r2QrD(Y@El!wa;Ja6G9Y1947P*DC|{9~nO& z*vDnnU!8(cV%HevsraF%Y%2{Z>CL0?64eu9r^t#WjW4~3uw8d}WHzsV%oq-T)Y z0-c!FWX5j1{1##?{aTeCW2b$PEnwe;t`VPCm@sQ`+$$L2=3kBR%2XU1{_|__XJ$xt zibjY2QlDVs)RgHH*kl&+jn*JqquF)k_Ypibo00lcc<2RYqsi-G%}k0r(N97H7JEn7@E3ZTH0JK>d8)E~A-D z!B&z9zJw0Bi^fgQZI%LirYaBKnWBXgc`An*qvO^*$xymqKOp(+3}IsnVhu?YnN7qz zNJxDN-JWd7-vIiv2M9ih>x3gNVY%DzzY~dCnA}76IRl!`VM=6=TYQ=o&uuE8kHqZT zoUNod0v+s9D)7aLJ|hVqL0li1hg)%&MAciI(4YJ=%D4H$fGQ&Lu-?@>>@pEgC;ERrL= zI^cS&3q8fvEGTJZgZwL5j&jp%j9U^Of6pR{wA^u=tVt#yCQepXNIbynGnuWbsC_EE zRyMFq{5DK692-*kyGy~An>AdVR9u___fzmmJ4;^s0yAGgO^h{YFmqJ%ZJ_^0BgCET zE6(B*SzeZ4pAxear^B-YW<%BK->X&Cr`g9_;qH~pCle# zdY|UB5cS<}DFRMO;&czbmV(?vzikf)Ks`d$LL801@HTP5@r><}$xp}+Ip`u_AZ~!K zT}{+R9Wkj}DtC=4QIqJok5(~0Ll&_6PPVQ`hZ+2iX1H{YjI8axG_Bw#QJy`6T>1Nn z%u^l`>XJ{^vX`L0 z1%w-ie!dE|!SP<>#c%ma9)8K4gm=!inHn2U+GR+~ zqZVoa!#aS0SP(|**WfQSe?cA=1|Jwk`UDsny%_y{@AV??N>xWekf>_IZLUEK3{Ksi zWWW$if&Go~@Oz)`#=6t_bNtD$d9FMBN#&97+XKa+K2C@I9xWgTE{?Xnhc9_KKPcujj@NprM@e|KtV_SR+ zSpeJ!1FGJ=Te6={;;+;a46-*DW*FjTnBfeuzI_=I1yk8M(}IwEIGWV0Y~wia;}^dg z{BK#G7^J`SE10z4(_Me=kF&4ld*}wpNs91%2Ute>Om`byv9qgK4VfwPj$`axsiZ)wxS4k4KTLb-d~!7I@^Jq`>?TrixHk|9 zqCX7@sWcVfNP8N;(T>>PJgsklQ#GF>F;fz_Rogh3r!dy*0qMr#>hvSua;$d z3TCZ4tlkyWPTD<=5&*bUck~J;oaIzSQ0E03_2x{?weax^jL3o`ZP#uvK{Z5^%H4b6 z%Kbp6K?>{;8>BnQy64Jy$~DN?l(ufkcs6TpaO&i~dC>0fvi-I^7YT#h?m;TVG|nba%CKRG%}3P*wejg) zI(ow&(5X3HR_xk{jrnkA-hbwxEQh|$CET9Qv6UpM+-bY?E!XVorBvHoU59;q<9$hK z%w5K-SK zWT#1OX__$ceoq0cRt>9|)v}$7{PlfwN}%Wh3rwSl;%JD|k~@IBMd5}JD#TOvp=S57 zae=J#0%+oH`-Av}a(Jqhd4h5~eG5ASOD)DfuqujI6p!;xF_GFcc;hZ9k^a7c%%h(J zhY;n&SyJWxju<+r`;pmAAWJmHDs{)V-x7(0-;E?I9FWK@Z6G+?7Py8uLc2~Fh1^0K zzC*V#P88(6U$XBjLmnahi2C!a+|4a)5Ho5>owQw$jaBm<)H2fR=-B*AI8G@@P-8I8 zHios92Q6Nk-n0;;c|WV$Q);Hu4;+y%C@3alP`cJ2{z~*m-@de%OKVgiWp;4Q)qf9n zJ!vmx(C=_>{+??w{U^Bh|LFJ<6t}Er<-Tu{C{dv8eb(kVQ4!fOuopTo!^x1OrG}0D zR{A#SrmN`=7T29bzQ}bwX8OUufW9d9T4>WY2n15=k3_rfGOp6sK0oj7(0xGaEe+-C zVuWa;hS*MB{^$=0`bWF(h|{}?53{5Wf!1M%YxVw}io4u-G2AYN|FdmhI13HvnoK zNS2fStm=?8ZpKt}v1@Dmz0FD(9pu}N@aDG3BY8y`O*xFsSz9f+Y({hFx;P_h>ER_& z`~{z?_vCNS>agYZI?ry*V96_uh;|EFc0*-x*`$f4A$*==p`TUVG;YDO+I4{gJGrj^ zn?ud(B4BlQr;NN?vaz_7{&(D9mfd z8esj=a4tR-ybJjCMtqV8>zn`r{0g$hwoWRUI3}X5=dofN){;vNoftEwX>2t@nUJro z#%7rpie2eH1sRa9i6TbBA4hLE8SBK@blOs=ouBvk{zFCYn4xY;v3QSM%y6?_+FGDn z4A;m)W?JL!gw^*tRx$gqmBXk&VU=Nh$gYp+Swu!h!+e(26(6*3Q!(!MsrMiLri`S= zKItik^R9g!0q7y$lh+L4zBc-?Fsm8`CX1+f>4GK7^X2#*H|oK}reQnT{Mm|0ar<+S zRc_dM%M?a3bC2ILD`|;6vKA`a3*N~(cjw~Xy`zhuY2s{(7KLB{S>QtR3NBQ3>vd+= z#}Q)AJr7Y_-eV(sMN#x!uGX08oE*g=grB*|bBs}%^3!RVA4f%m3=1f0K=T^}iI&2K zuM2GG5_%+#v-&V>?x4W9wQ|jE2Q7Be8mOyJtZrqn#gXy-1fF1P$C8+We&B*-pi#q5 zETp%H6g+%#sH+L4=ww?-h;MRCd2J9zwQUe4gHAbCbH08gDJY;F6F)HtWCRW1fLR;)ysGZanlz*a+|V&@(ipWdB!tz=m_0 z6F}`d$r%33bw?G*azn*}Z;UMr{z4d9j~s`0*foZkUPwpJsGgoR0aF>&@DC;$A&(av z?b|oo;`_jd>_5nye`DVOcMLr-*Nw&nA z82E8Dw^$Lpso)gEMh?N|Uc^X*NIhg=U%enuzZOGi-xcZRUZmkmq~(cP{S|*+A6P;Q zprIkJkIl51@ng)8cR6QSXJtoa$AzT@*(zN3M+6`BTO~ZMo0`9$s;pg0HE3C;&;D@q zd^0zcpT+jC%&=cYJF+j&uzX87d(gP9&kB9|-zN=69ymQS9_K@h3ph&wD5_!4q@qI@ zBMbd`2JJ2%yNX?`3(u&+nUUJLZ=|{t7^Rpw#v-pqD2_3}UEz!QazhRty%|Q~WCo7$ z+sIugHA%Lmm{lBP#bnu_>G}Ja<*6YOvSC;89z67M%iG0dagOt1HDpDn$<&H0DWxMU zxOYaaks6%R@{`l~zlZ*~2}n53mn2|O&gE+j*^ypbrtBv{xd~G(NF?Z%F3>S6+qcry z?ZdF9R*a;3lqX_!rI(Cov8ER_mOqSn6g&ZU(I|DHo7Jj`GJ}mF;T(vax`2+B8)H_D zD0I;%I?*oGD616DsC#j0x*p+ZpBfd=9gR|TvB)832CRhsW_7g&WI@zp@r7dhg}{+4f=(cO2s+)jg0x(*6|^+6W_=YIfSH0lTcK* z%)LyaOL6em@*-_u)}Swe8rU)~#zT-vNiW(D*~?Zp3NWl1y#fo!3sK-5Ek6F$F5l3| zrFFD~WHz1}WHmzzZ!n&O8rTgfytJG*7iE~0`0;HGXgWTgx@2fD`oodipOM*MOWN-} zJY-^>VMEi8v23ZlOn0NXp{7!QV3F1FY_URZjRKMcY(2PV_ms}EIC^x z=EYB5UUQ{@R~$2Mwiw$_JAcF+szKB*n(`MYpDCl>~ss54uDQ%Xf-8|dgO zY)B_qju=IaShS|XsQo=nSYxV$_vQR@hd~;qW)TEfU|BA0&-JSwO}-a*T;^}l;MgLM zz}CjPlJX|W2vCzm3oHw3vqsRc3RY=2()}iw_k2#eKf&VEP7TQ;(DDzEAUgj!z_h2Br;Z3u=K~LqM6YOrlh)v9`!n|6M-s z?XvA~y<5?WJ{+yM~uPh7uVM&g-(;IC3>uA}ud?B3F zelSyc)Nx>(?F=H88O&_70%{ATsLVTAp88F-`+|egQ7C4rpIgOf;1tU1au+D3 zlz?k$jJtTOrl&B2%}D}8d=+$NINOZjY$lb{O<;oT<zXoAp01KYG$Y4*=)!&4g|FL(!54OhR-?)DXC&VS5E|1HGk8LY;)FRJqnz zb_rV2F7=BGwHgDK&4J3{%&IK~rQx<&Kea|qEre;%A~5YD6x`mo>mdR)l?Nd%T2(5U z_ciT02-zt_*C|vn?BYDuqSFrk3R(4B0M@CRFmG{5sovIq4%8AhjXA5UwRGo)MxZlI zI%vz`v8B+#ff*XtGnciczFG}l(I}{YuCco#2E6|+5WJ|>BSDfz0oT+F z%QI^ixD|^(AN`MS6J$ zXlKNTFhb>KDkJp*4*LaZ2WWA5YR~{`={F^hwXGG*rJYQA7kx|nwnC58!eogSIvy{F zm1C#9@$LhK^Tl>&iM0wsnbG7Y^MnQ=q))MgApj4)DQt!Q5S`h+5a%c7M!m%)?+h65 z0NHDiEM^`W+M4)=q^#sk(g!GTpB}edwIe>FJQ+jAbCo#b zXmtd3raGJNH8vnqMtjem<_)9`gU_-RF&ZK!aIenv7B2Y0rZhon=2yh&VsHzM|`y|0x$Zez$bUg5Nqj?@~^ zPN43MB}q0kF&^=#3C;2T*bDBTyO(+#nZnULkVy0JcGJ36or7yl1wt7HI_>V7>mdud zv2II9P61FyEXZuF$=69dn%Z6F;SOwyGL4D5mKfW)q4l$8yUhv7|>>h_-4T*_CwAyu7;DW}_H zo>N_7Gm6eed=UaiEp_7aZko@CC61@(E1be&5I9TUq%AOJW>s^9w%pR5g2{7HW9qyF zh+ZvX;5}PN0!B4q2FUy+C#w5J?0Tkd&S#~94(AP4%fRb^742pgH7Tb1))siXWXHUT z1Wn5CG&!mGtr#jq6(P#!ck@K+FNprcWP?^wA2>mHA03W?kj>5b|P0ErXS) zg2qDTjQ|grCgYhrH-RapWCvMq5vCaF?{R%*mu}1)UDll~6;}3Q*^QOfj!dlt02lSzK z?+P)02Rrq``NbU3j&s*;<%i4Y>y9NK&=&KsYwvEmf5jwTG6?+Pu1q9M8lLlx)uZZ7 zizhr~e0ktGs-=$li-2jz^_48-jk**y&5u0`B2gc#i$T1~t+AS*kEfR*b{^Ec>2-F~ zKYRl&uQ5yO@EtAZX8ZSqx;8+AKf+CqhlUSpp*VfyBMv+%wxN5GukZEi^_to%MFRc0 zdXqJ*jk?#uYT6EJe446@(f6G4vhnxQP|pGeJ?-#|Ksq?g*ky=}x+Qnx+!<>Y(XStN zQIND`{KU}&l)E*ntI^}kJ=ly8DML{!(58Xk4_bzIc@v~e;>wKl_`7G%pGz~4KH*CTp;_|52)d!+ximd$|8v@zzEq%j68QXkgf$7eM~xdM5q5i z{?qFx_W|eq@L03bWJfjy^z@()-iCjzjREuf zb_a(yTz)ZKWCF%Lp>^2-%Q?*t{06}x#DLN3cO=i>h6#-a`z;<5rBGGM6GA(WqvRcX%Pn?Uvs1#e|ePSNJEC%+X(YI$x)`s$%>O#%}D9dgqWfq4yfVz^%FglokdFR}uJQhx|}_w`9Ulx38Ha>ZslKs58c-@IFI&f;?xM zbK>rKNfPFsf>%+k6%(A6=7Aac^_qrOCNqb3ZVJ;8pt!?1DR*ynJb#@II9h?)xB)A~ zm9Kk)Hy}!Z+W}i6ZJDy+?yY_=#kWrzgV)2eZAx_E=}Nh7*#<&mQz`Umfe$+l^P(xd zN}PA2qII4}ddCU+PN+yxkH%y!Qe(;iH3W%bwM3NKbU_saBo<8x9fGNtTAc_SizU=o zC3n2;c%LoU^j90Sz>B_p--Fzqv7x7*?|~-x{haH8RP)p|^u$}S9pD-}5;88pu0J~9 zj}EC`Q^Fw}`^pvAs4qOIuxKvGN@DUdRQ8p-RXh=3S#<`3{+Qv6&nEm)uV|kRVnu6f zco{(rJaWw(T0PWim?kkj9pJ)ZsUk9)dSNLDHf`y&@wbd;_ita>6RXFJ+8XC*-wsiN z(HR|9IF283fn=DI#3Ze&#y3yS5;!yoIBAH(v}3p5_Zr+F99*%+)cp!Sy8e+lG?dOc zuEz<;3X9Z5kkpL_ZYQa`sioR_@_cG z8tT~GOSTWnO~#?$u)AcaBSaV7P~RT?Nn8(OSL1RmzPWRWQ$K2`6*)+&7^zZBeWzud z*xb3|Fc~|R9eH+lQ#4wF#c;)Gka6lL(63C;>(bZob!i8F-3EhYU3|6-JBC0*5`y0| zBs!Frs=s!Sy0qmQNgIH|F`6(SrD1js2prni_QbG9Sv@^Pu2szR9NZl8GU89gWWvVg z2^-b*t+F{Nt>v?js7hnlC`tRU(an0qQG7;h6T~ z-`vf#R-AE$pzk`M{gCaia}F`->O2)60AuGFAJg> z*O2IZqTx=AzDvC49?A92>bQLdb&32_4>0Bgp0ESXXnd4B)!$t$g{*FG%HYdt3b3a^J9#so%BJMyr2 z{y?rzW!>lr097b9(75#&4&@lkB1vT*w&0E>!dS+a|ZOu6t^zro2tiP)bhcNNxn zbJs3_Fz+?t;4bkd8GfDI7ccJ5zU`Bs~ zN~bci`c`a%DoCMel<-KUCBdZRmew`MbZEPYE|R#|*hhvhyhOL#9Yt7$g_)!X?fK^F z8UDz)(zpsvriJ5aro5>qy`Fnz%;IR$@Kg3Z3EE!fv9CAdrAym6QU82=_$_N5*({_1 z7!-=zy(R{xg9S519S6W{HpJZ8Is|kQ!0?`!vxDggmslD59)>iQ15f z7J8NqdR`9f8H|~iFGNsPV!N)(CC9JRmzL9S}7U-K@`X893f3f<8|8Ls!^eA^#(O6nA+ByFIXcz_WLbfeG|nHJ5_sJJ^gNJ%SI9#XEfNRbzV+!RkI zXS$MOVYb2!0vU}Gt7oUy*|WpF^*orBot~b2J@^be?Gq;U%#am8`PmH-UCFZ&uTJlnetYij0z{K1mmivk$bdPbLodu;-R@@#gAV!=d%(caz$E?r zURX0pqAn7UuF6dULnoF1dZ$WM)tHAM{eZK6DbU1J`V5Dw<;xk}Nl`h+nfMO_Rdv z3SyOMzAbYaD;mkxA7_I_DOs#Bk;e5D%gsS3q)hlmi1w{FsjKNJE22`AjmNiAPRnIc zcIkN25;rOn3FipAFd(PnlK9{03w6Q<(68#1Jw`{axEGQE{Ac>^U$h);h2ADICmaNxrfpb`Jdr*)Y1SicpYKCFv$3vf~;5aW>n^7QGa63MJ z;B1+Z>WQ615R2D8JmmT`T{QcgZ+Kz1hTu{9FOL}Q8+iFx-Vyi}ZVVcGjTe>QfA`7W zFoS__+;E_rQIQxd(Bq4$egKeKsk#-9=&A!)(|hBvydsr5ts0Zjp*%*C0lM2sIOx1s zg$xz?Fh?x!P^!vWa|}^+SY8oZHub7f;E!S&Q;F?dZmvBxuFEISC}$^B_x*N-xRRJh zn4W*ThEWaPD*$KBr8_?}XRhHY7h^U1aN6>m=n~?YJQd8+!Uyq_3^)~4>XjelM&!c9 zCo|0KsGq7!KsZ~9@%G?i>LaU7#uSTMpypocm*oqJHR|wOgVWc7_8PVuuw>x{kEG4T z$p^DV`}jUK39zqFc(d5;N+M!Zd3zhZN&?Ww(<@AV-&f!v$uV>%z+dg9((35o@4rqLvTC-se@hkn^6k7+xHiK-vTRvM8{bCejbU;1@U=*r}GTI?Oc$!b6NRcj83-zF; z=TB#ESDB`F`jf4)z=OS76Se}tQDDHh{VKJk#Ad6FDB_=afpK#pyRkGrk~OuzmQG)} z*$t!nZu$KN&B;|O-aD=H<|n6aGGJZ=K9QFLG0y=Jye_ElJFNZJT;fU8P8CZcLBERjioAOC0Vz_pIXIc};)8HjfPwNy zE!g|lkRv3qpmU?shz(BBt5%TbpJC3HzP9!t7k*Fh48!-HlJ4TTgdCr3rCU!iF}kgu z4Qs;K@XOY~4f~N}Jl8V_mGbwzvNLbl&0e9UG4W;kvjTK|5`-Ld+eQ6YRF`N0ct%u% z^3J_{7r#_W1zm|>IPN!yWCRrN)N!7v`~ptNkIXKipQ6ogFvcnI5ugxdoa{d;uD67g zgo^}QuZRkB540Vc!@c80(wFG=$ct}oHq(#W0+-XX(;Rrt`x=<45X}ficNtI2(&}=~ zb(!}tNz?s`wm{gK?2tdf+OEF;tzx<(3fMd7_tM@Ghs$Z(Os-H(kYq#qB|J-aC9Ku?fsWwJhB36c)A zu|a7ZF?V8X7l2g5~xqZf>2=6Dsi5lfo zKIRL&@MLJyaBE)V_9=pJYu%U2wxR*-(0MI5_|yqP`?h@cks(5LR@XUKLMI_xuVtiu zRvpDS8MyUMRFM6`P+Sjc!A_e^H38Qu7b{b7QZ>NHyA6k-YYygQuW&C_OGO(7V7?}r)zedSVpBI zuk29Z4GW3C0GpfozbZQya454sjt@ndQmsp=DA&@sWw&xmOlDk1JIcMNp~-ES$&A~k zG#W(6hBj?!Fu8Q4WYexoSBa8_5=v20xnx6H?e;$t)5|f&{7=vOye^&3_c-Ug?|a@e z=X`&qT_5B7N9vZoPBhXOTEDV;4&x2Je4}T(UB~O-$D#CjX77$R?RZ*`ed~$G;$4YS z4n*|Pop(!NN79Hk2}U#cfEEwdxM)xQm}$~rV03xc=#U@@Y*}qEmot5KvDb=8{!E-n zl4p?}&g2h^sUGyTcGh=0aQzQb*k;K;dvbeZUgmwEv>%#(EPtj=gHKdi|E8@w+|>KC zxEU>b>P+9Xf}pEyQK(}#QrBG4Jaf!iE!qpMbTu>gb!gtdq<`@xO+roQl+S_7)!G(% zdy)$iGmJ1cwP?F=IyyV1-$|kf|EKM3B@I&lZ%NI@VV;*mQdLWjc#t|Vbk_Q~>&O03 zIcSr$(qLAINj7a z;!||v&1D5SX#X@5jNd}jUsi-CH_Scjyht&}q2p*CJCC-`&NyXf)vD5{e!HO629D-O z%bZelTcq=DoRX>zeWCa^RmR3*{x9;3lZ75M#S)!W0bRIFH#P6b%{|HRSZ5!!I#s)W z_|XXZQ<0_`>b^^0Z>LU64Yg1w)8}#M^9se(OZ9~baZ7fsKFc;EtnB>kesci#>=icG zuHdjax2^=!_(9?0l7;G7^-}9>Y#M zm;9*GT~dBuYWdk49%mZM0=H#FY1)}7NE5DE_vsqrA0`?0R0q535qHjWXcl|gz9Fq$ zMKxgL;68l!gm3y0durIr3LHv~y*ABm` zYhQG0UW#hg@*A{&G!;$FS43}rIF$e6yRdGJWVR<}uuJ_5_8qa3xaHH^!VzUteVp;> z<0`M>3tnY$ZFb$(`0sg93TwGyP;`9UYUWxO&CvAnSzei&ap))NcW;R`tA=y^?mBmG+M*&bqW5kL$V(O;(p)aEk`^ci?2Jwxu>0sy>a7+Wa9t z5#I2o;+gr^9^&km^z7>xJWbN&Ft>Vna34E zI@BBzwX)R}K3SL?)enrDJ45QLt;-7CFJk{`cF3L4Z^CtG_r5)0)HV>BOYPIUh#D%| zYQAu31f{bm-D*`_k7DTTr?Nkw_gY%J1cb2&TdtibY?V=|SSIOlA;|5C!2@?YQ z-$?G0jj^mG|MP>DmbF7}T~C$H6=CpZ~hd zZ1C|xV@=h#^~`3LSCnmI(vZ|5r3>eq5*UB)dhdy``*gKY3Eg%jSK8I-`G+OWWlD)T zt$wSQ=||lSkiKy}YF-k}@W9EiS?)z`hK{R!dd-$BCJvBtAN-yXn3njU$MisEtp!?Q z%Vk-*(wy9dd15(-WFw_&^tT;;IpF?ox1`Qq3-0zVTk+$W_?q}GfAQlPcrB^?&tWSI z2BB!K=sH7FUYmXa_dcV^Z3>5z8}~W{S!$jVR_3hu_|wl2|gmRH8ftn^z@fW75*;-`;wU+fY+BR_yx6BZnE5_Hna({jrPiubRp$jZ=T=t$hx&NeCV1!vuCcl4PJ0p0Fjp>6K} zHkoD1gQk=P2hYcT%)cJ2Q5WuA|5_x+dX0%hnozfTF>$#Wz~X!MY>){H4#fB#7^ID* z1*o2Hzp}?WVs&gbS?Uq(CT0sP+F)u9{xfgg6o_{8J#m;|NeJqDHhb(Q8%z8aM_qeM zn83>d`uDd47WIuKp78JBYo2SYupGcNXIzeou^eMY`@%Bv8elZ>q~3uq#~IX)g%g;h zoUXymEd>|kVsMkyb&1l~lrE-`w(0PObapYa35DJ4Y03Jv_!DKp}0HTbOgZRM=;PSsuAJJJ1 zItc+tu9;ANG;qHaCI|T85!euhFK~VK^G2LZV1+cbzS?>ar@>emg;JTI5VAn1g5U~| zU=p&k0OlSzc$U=s#9_uL3&n|6A1X$XvrE9vFV@`A4G#!D1QcFCeE`F2N(deJx>)*A z$XIW0P~-NbAd=5i6`s<~(vAQX9t$dbVqc5|E|CHRtb$1(l&KSNh_t2#k_l95KnP86 z)ns_DGspv-M0z0#h2a+*oH|{5~j{ zXGD=}cLrBSESQ0u$XmQlFfWMCAWaS;wKK%#aSSYK=qljBiY(s zT$v;We24&$w=avIILsMt0%1fDyah|AlLNg#WL$Lu)tf}YfqO%+pH~QC*bZO4aM*i9 zrPFf|5!hv@XY8CzaFh*Dy9vH|2fKKr(@x}`L#9^*vOae|lk`adG#oZZAyk|TOV8`9L zc-sQu%y1MQes&J?)a1}Zc*>-P!6j-T#75V$lLC!TuMB(!G-+D2;XptUxymSPFI-K&0x}B1?h$ z3-9**-9!);fwyiWB5gS$i;P~c=^}5-6G@{4TWDBRDc6(M|%qa-mS`z`u9kWo{Xl_uc;hXOkRd literal 0 HcmV?d00001 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..2a563242 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.2-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 00000000..fbd7c515 --- /dev/null +++ b/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 00000000..a9f778a7 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,104 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega From c29348da281855cc966e979213fb06b10ea95cca Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 26 Jul 2020 08:40:46 +0200 Subject: [PATCH 126/512] gradle build --- build.gradle.kts | 67 +++++++++++++++++++++++++++++++++++++++++++++ settings.gradle.kts | 1 + 2 files changed, 68 insertions(+) create mode 100644 build.gradle.kts create mode 100644 settings.gradle.kts diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 00000000..a4485e72 --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,67 @@ +plugins { + id("java") + id("maven-publish") +} + +repositories { + mavenLocal() + mavenCentral() +} + +dependencies { + implementation("org.java-websocket:Java-WebSocket:1.5.1") + implementation("org.jsoup:jsoup:1.8.1") + implementation("org.json:json:20190722") + implementation("commons-configuration:commons-configuration:1.7") + implementation("log4j:log4j:1.2.17") + implementation("commons-cli:commons-cli:1.2") + implementation("commons-io:commons-io:1.3.2") + implementation("org.apache.httpcomponents:httpclient:4.3.6") + implementation("org.apache.httpcomponents:httpmime:4.3.3") + implementation("org.graalvm.js:js:20.1.0") + testImplementation(enforcedPlatform("org.junit:junit-bom:5.6.2")) + testImplementation("org.junit.jupiter:junit-jupiter") + testImplementation("junit:junit:4.13") +} + +group = "com.rarchives.ripme" +version = "1.7.94" +description = "ripme" + +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} + +tasks.withType { + manifest { + attributes["Main-Class"] = "com.rarchives.ripme.App" + } + + // To add all of the dependencies otherwise a "NoClassDefFoundError" error + from(sourceSets.main.get().output) + + dependsOn(configurations.runtimeClasspath) + from({ + configurations.runtimeClasspath.get().filter { it.name.endsWith("jar") }.map { zipTree(it) } + }) +} + +publishing { + publications { + create("maven") { + from(components["java"]) + } + } +} + +tasks.withType { + options.encoding = "UTF-8" +} + +tasks.test { + useJUnitPlatform { + includeEngines("junit-jupiter") + includeEngines("junit-vintage") + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 00000000..25d89451 --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "ripme" From 4da5b446439bc865d51024d231c4a72b7d2e1f8d Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 26 Jul 2020 19:09:15 +0200 Subject: [PATCH 127/512] gradle, make all archive tasks reproducible --- build.gradle.kts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index a4485e72..82b02113 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -65,3 +65,10 @@ tasks.test { includeEngines("junit-vintage") } } + +// make all archive tasks in the build reproducible +tasks.withType().configureEach { + isPreserveFileTimestamps = false + isReproducibleFileOrder = true +} + From 7ad1df97c085e19e6d4af4d5807f40e9e4ab5458 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 26 Jul 2020 19:41:16 +0200 Subject: [PATCH 128/512] gradle, produce jacoco coverage report, html --- build.gradle.kts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index 82b02113..d844c0ea 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,4 +1,5 @@ plugins { + id("jacoco") id("java") id("maven-publish") } @@ -64,6 +65,7 @@ tasks.test { includeEngines("junit-jupiter") includeEngines("junit-vintage") } + finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run } // make all archive tasks in the build reproducible @@ -72,3 +74,12 @@ tasks.withType().configureEach { isReproducibleFileOrder = true } +tasks.jacocoTestReport { + dependsOn(tasks.test) // tests are required to run before generating the report + reports { + xml.isEnabled = false + csv.isEnabled = false + html.destination = file("${buildDir}/jacocoHtml") + } +} + From 8f17f952228f9139116c341f73b799ac486e0c56 Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 30 Jul 2020 07:21:13 +0200 Subject: [PATCH 129/512] gradle, not execute slow tests by default ```gradle slowTests``` will execute only the tests with tag slow. --- build.gradle.kts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index d844c0ea..b4435c29 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -62,12 +62,20 @@ tasks.withType { tasks.test { useJUnitPlatform { + // gradle-6.5.1 not yet allows passing this as parameter, so exclude it + excludeTags("slow") includeEngines("junit-jupiter") includeEngines("junit-vintage") } finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run } +tasks.register("slowTests") { + useJUnitPlatform { + includeTags("slow") + } +} + // make all archive tasks in the build reproducible tasks.withType().configureEach { isPreserveFileTimestamps = false From 54d2ff4b46474f43688f951ca56b3a835c04ac11 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 21 Nov 2020 11:08:59 +0100 Subject: [PATCH 130/512] gradle, not execute flaky tests by default --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index b4435c29..5572205f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -63,7 +63,7 @@ tasks.withType { tasks.test { useJUnitPlatform { // gradle-6.5.1 not yet allows passing this as parameter, so exclude it - excludeTags("slow") + excludeTags("flaky","slow") includeEngines("junit-jupiter") includeEngines("junit-vintage") } From 1742995ea33d7f47776e026fdd9bbed71080899d Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 14 Feb 2021 12:23:15 +0100 Subject: [PATCH 131/512] add gradle commands into readme --- README.md | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1b2525c5..3d09638a 100644 --- a/README.md +++ b/README.md @@ -77,14 +77,21 @@ If you're a developer, you can add your own Ripper by following the wiki guide: # Compiling & Building -The project uses [Maven](http://maven.apache.org/). -To build the .jar file using Maven, navigate to the root project directory and run: +The project uses [Gradle](https://gradle.org) or [Maven](http://maven.apache.org/). +Therefor both commands are given. To build the .jar file, navigate to the root +project directory and run: ```bash mvn clean compile assembly:single +mvn -B package assembly:single -Dmaven.test.skip=true +``` +```bash +./gradlew clean build +./gradlew clean build -x test --warning-mode all ``` -This will include all dependencies in the JAR. +This will include all dependencies in the JAR. One can skip executing the tests +as well. # Running Tests @@ -98,6 +105,12 @@ mvn test -DexcludedGroups= -Dgroups=flaky,slow mvn test '-Dgroups=!slow' ``` +```bash +./gradlew test +./gradlew test -DexcludeTags= -DincludeTags=flaky,slow +./gradlew test '-DincludeTags=!slow' +``` + Please note that some tests may fail as sites change and our rippers become out of date. Start by building and testing a released version of RipMe and then ensure that any changes you make do not cause more tests to break. From bfa50147aba114a05f3e2a77595dfbf756af270e Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 22 Feb 2021 02:41:39 +0100 Subject: [PATCH 132/512] gradle, set jar version to -- set the version to the last tag, plus the number of commits since the last tag (called "distance"), and the git hash (sha). an example: 1.7.94-9-c9d7deca in case there is an annotated tag on the commit built itself it is a release, and no distance, and sha is added: 1.7.94 in case the build is not from main and master branch, the branch name will be added, special characters are translated to underscore: 1.7.94-9-c9d7deca-other_branch --- build.gradle.kts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 5572205f..0cdfc10c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,4 +1,5 @@ plugins { + id("fr.brouillard.oss.gradle.jgitver") version "0.9.1" id("jacoco") id("java") id("maven-publish") @@ -29,9 +30,15 @@ group = "com.rarchives.ripme" version = "1.7.94" description = "ripme" -java { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 +jgitver { + gitCommitIDLength = 8 + nonQualifierBranches = "main,master" + useGitCommitID = true +} + +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 } tasks.withType { From c2a80db78bad475395519c591d904bf99cc08abf Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 23 Feb 2021 19:24:32 +0100 Subject: [PATCH 133/512] maven github action now only manually triggered --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index ea6ac25a..038c890f 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -1,6 +1,6 @@ name: Java CI -on: [push, pull_request] +on: workflow_dispatch jobs: build: From 1847965a452ddbcbb542c3a67422a1851f4a358c Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 23 Feb 2021 19:19:02 +0100 Subject: [PATCH 134/512] build with gradle, cache build artefacts to speed it up --- .github/workflows/gradle.yml | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/gradle.yml diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml new file mode 100644 index 00000000..38ad74fa --- /dev/null +++ b/.github/workflows/gradle.yml @@ -0,0 +1,44 @@ +name: CI + release + +on: [push, pull_request] + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + java: [1.8] + include: # test newest java on one os only, upload from ubuntu java8 + - os: ubuntu-latest + java: 1.15 + - os: ubuntu-latest + upload: true + + steps: + - uses: actions/checkout@v1 + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - name: Cache Gradle packages + # speed up the build by caching dependencies, downloaded versions + uses: actions/cache@v2 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + - name: Build with Gradle + run: ./gradlew build + - name: upload jar as asset + if: matrix.upload + uses: actions/upload-artifact@v2 + with: + name: zipped-ripme-jar + path: build/libs/*.jar + +# vim:set ts=2 sw=2 et: From 0750558cac1db9aa3c9524eb9c45dc3b5b97beb1 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Feb 2021 10:59:09 +0100 Subject: [PATCH 135/512] gradle task to run all tests including flaky, slow --- build.gradle.kts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 0cdfc10c..30ee1206 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -77,9 +77,9 @@ tasks.test { finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run } -tasks.register("slowTests") { +tasks.register("testAll") { useJUnitPlatform { - includeTags("slow") + includeTags("any()", "none()") } } From cbe8fbb3da3ebe531625409d3c16d0fb6f468ece Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Feb 2021 10:59:49 +0100 Subject: [PATCH 136/512] cyberdropripper fails sometimes, flaky --- .../rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java index 4d077628..847f2abf 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java @@ -4,6 +4,7 @@ import com.rarchives.ripme.ripper.rippers.CyberdropRipper; import com.rarchives.ripme.utils.Http; import org.jsoup.nodes.Document; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -29,6 +30,7 @@ public class CyberdropRipperTest extends RippersTest { } @Test + @Tag("flaky") public void testCyberdropNumberOfFiles() throws IOException { List testURLs = new ArrayList(); From 7155804a5a8eeb1fd3e982e9bbdce02d6028058e Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Feb 2021 11:17:49 +0100 Subject: [PATCH 137/512] gradle-6.8.3 --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 2a563242..442d9132 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 6476611101e8ac277e6a199e16a5d264bcfd679b Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Feb 2021 11:26:00 +0100 Subject: [PATCH 138/512] hentainexus flaky --- .../ripme/tst/ripper/rippers/HentainexusRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java index a244276c..00340eba 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java @@ -8,10 +8,12 @@ import java.util.List; import com.rarchives.ripme.ripper.rippers.HentaiNexusRipper; import org.json.JSONObject; import org.junit.Assert; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class HentainexusRipperTest extends RippersTest { @Test + @Tag("flaky") public void testHentaiNexusJson() throws IOException { List testURLs = new ArrayList<>(); testURLs.add(new URL("https://hentainexus.com/view/9202")); From 9db885dbcd17c155e2c5685df32b65fffff0f111 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Feb 2021 15:05:25 +0100 Subject: [PATCH 139/512] ArtStationRipperTest flaky --- .../ripme/tst/ripper/rippers/ArtStationRipperTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java index d7cf6cdf..e29a32ed 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java @@ -7,12 +7,13 @@ import java.util.List; import com.rarchives.ripme.ripper.rippers.ArtStationRipper; -import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class ArtStationRipperTest extends RippersTest { @Test + @Tag("flaky") public void testArtStationProjects() throws IOException { List contentURLs = new ArrayList<>(); contentURLs.add(new URL("https://www.artstation.com/artwork/the-dwarf-mortar")); @@ -25,7 +26,7 @@ public class ArtStationRipperTest extends RippersTest { } @Test - @Disabled("Failed with cloudflare protection") + @Tag("flaky") public void testArtStationUserProfiles() throws IOException { List contentURLs = new ArrayList<>(); contentURLs.add(new URL("https://www.artstation.com/heitoramatsu")); From 1f2cb3d6b90be189a1c0b103a83ee06aea4ed385 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Feb 2021 17:24:46 +0100 Subject: [PATCH 140/512] imgboxrppertest flaky --- .../rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java index 3b6bb782..9e3b6b32 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java @@ -5,10 +5,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.ImgboxRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class ImgboxRipperTest extends RippersTest { @Test + @Tag("flaky") public void testImgboxRip() throws IOException { ImgboxRipper ripper = new ImgboxRipper(new URL("https://imgbox.com/g/FJPF7t26FD")); testRipper(ripper); From 2347e74970f3466d71592f4263c0920d92bbf13c Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Feb 2021 15:26:27 +0100 Subject: [PATCH 141/512] set version into Implementation-Version, MANIFEST.MF part of #12 --- build.gradle.kts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index 0cdfc10c..a31ee0fd 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -44,6 +44,8 @@ java { tasks.withType { manifest { attributes["Main-Class"] = "com.rarchives.ripme.App" + attributes["Implementation-Version"] = archiveVersion + } // To add all of the dependencies otherwise a "NoClassDefFoundError" error From f841a10b16d6469efff3cc61ebfdfbf1ed09596f Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 6 Mar 2021 05:54:53 +0100 Subject: [PATCH 142/512] development build, pre-release, #12 --- .github/workflows/gradle.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 38ad74fa..fb67e7df 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: build: - + runs-on: ${{ matrix.os }} strategy: matrix: @@ -17,11 +17,14 @@ jobs: upload: true steps: + - uses: actions/checkout@v1 + - name: Set up JDK uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} + - name: Cache Gradle packages # speed up the build by caching dependencies, downloaded versions uses: actions/cache@v2 @@ -32,8 +35,10 @@ jobs: key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} restore-keys: | ${{ runner.os }}-gradle- + - name: Build with Gradle run: ./gradlew build + - name: upload jar as asset if: matrix.upload uses: actions/upload-artifact@v2 @@ -41,4 +46,16 @@ jobs: name: zipped-ripme-jar path: build/libs/*.jar + - name: create pre-release + id: create-pre-release + if: matrix.upload + uses: "marvinpinto/action-automatic-releases@latest" + with: + repo_token: "${{ secrets.GITHUB_TOKEN }}" + automatic_release_tag: "latest" + prerelease: true + title: "development build" + files: | + build/libs/*.jar + # vim:set ts=2 sw=2 et: From 1a1f31db2c65fbdfcfdfb1cb252255a6b66b850b Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Feb 2021 17:08:24 +0100 Subject: [PATCH 143/512] 1.7.94-10-asdf is newer than 1.7.94 part of #12 --- README.md | 10 +++++---- .../com/rarchives/ripme/ui/UpdateUtils.java | 22 +++++++++++++------ .../rarchives/ripme/ui/UpdateUtilsTest.java | 17 ++++++++++++++ 3 files changed, 38 insertions(+), 11 deletions(-) create mode 100644 src/test/java/com/rarchives/ripme/ui/UpdateUtilsTest.java diff --git a/README.md b/README.md index be773987..7514e697 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,13 @@ RipMe is an album ripper for various websites. It is a cross-platform tool that ## Downloads -Download `ripme.jar` from the [latest release](https://github.com/ripmeapp/ripme/releases). +Download `ripme.jar` from the [latest release](/releases). For information about running the `.jar` file, see +[the How To Run wiki](https://github.com/ripmeapp/ripme/wiki/How-To-Run-RipMe). -**Note: If you're currently using version 1.2.x, 1.3.x or 1.7.49, you will not automatically get updates to the newest versions. We recommend downloading the latest version from the link above.** - -For information about running the `.jar` file, see [the How To Run wiki](https://github.com/ripmeapp/ripme/wiki/How-To-Run-RipMe). +The version number like ripme-1.7.94-17-2167aa34-feature_auto_release.jar contains a release number (1.7.94), given by +a person the number of commits since this version (17). The commit SHA (2167aa34) is there uniquely referencing the +source code ripme was built from. If it is not built from the main branch, the branch name (feature/auto-release) is +given. ## Installation diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 8c1b415e..d2379940 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -23,7 +23,8 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.94"; + // do not update the default version without adjusting the unit test. the real version comes from METAINF.MF + private static final String DEFAULT_VERSION = "1.7.94-10-b6345398"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static String mainFileName; @@ -165,7 +166,7 @@ public class UpdateUtils { } } - private static boolean isNewerVersion(String latestVersion) { + static boolean isNewerVersion(String latestVersion) { // If we're testing the update utils we want the program to always try to update if (Utils.getConfigBoolean("testing.always_try_to_update", false)) { logger.info("isNewerVersion is returning true because the key \"testing.always_try_to_update\" is true"); @@ -194,11 +195,18 @@ public class UpdateUtils { } private static int[] versionStringToInt(String version) { - String strippedVersion = version.split("-")[0]; - String[] strVersions = strippedVersion.split("\\."); - int[] intVersions = new int[strVersions.length]; - for (int i = 0; i < strVersions.length; i++) { - intVersions[i] = Integer.parseInt(strVersions[i]); + // a version string looks like 1.7.94, 1.7.94-10-something + // 10 is the number of commits since the 1.7.94 tag, so newer + // the int array returned then contains e.g. 1.7.94.0 or 1.7.94.10 + String[] strVersions = version.split("[\\.-]"); + // not consider more than 4 components of version, loop only the real number + // of components or maximum 4 components of the version string + int[] intVersions = new int[4]; + for (int i = 0; i < Math.min(4,strVersions.length); i++) { + // if it is an integer, set it, otherwise leave default 0 + if (strVersions[i].matches("\\d+")) { + intVersions[i] = Integer.parseInt(strVersions[i]); + } } return intVersions; } diff --git a/src/test/java/com/rarchives/ripme/ui/UpdateUtilsTest.java b/src/test/java/com/rarchives/ripme/ui/UpdateUtilsTest.java new file mode 100644 index 00000000..2f9ba697 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/ui/UpdateUtilsTest.java @@ -0,0 +1,17 @@ +package com.rarchives.ripme.ui; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class UpdateUtilsTest { + + @Test + public void testIsNewerVersion() { + UpdateUtils updateUtils = new UpdateUtils(); + Assertions.assertFalse(updateUtils.isNewerVersion("1.7.94")); + Assertions.assertFalse(updateUtils.isNewerVersion("1.7.94-9-asdf")); + Assertions.assertTrue(updateUtils.isNewerVersion("1.7.94-11-asdf")); + Assertions.assertTrue(updateUtils.isNewerVersion("1.7.95")); + } + +} \ No newline at end of file From e89e4875337fac147f9ccbea2e5fbf3f4fd9e1b9 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Feb 2021 17:11:41 +0100 Subject: [PATCH 144/512] cleanup ripstatusmessagetest, labelsbundlestest --- src/test/java/com/rarchives/ripme/tst/ui/LabelsBundlesTest.java | 1 - .../java/com/rarchives/ripme/tst/ui/RipStatusMessageTest.java | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ui/LabelsBundlesTest.java b/src/test/java/com/rarchives/ripme/tst/ui/LabelsBundlesTest.java index 6189d86a..77a590ce 100644 --- a/src/test/java/com/rarchives/ripme/tst/ui/LabelsBundlesTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ui/LabelsBundlesTest.java @@ -11,7 +11,6 @@ import java.util.Set; import com.rarchives.ripme.utils.Utils; import org.apache.log4j.Logger; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class LabelsBundlesTest { diff --git a/src/test/java/com/rarchives/ripme/tst/ui/RipStatusMessageTest.java b/src/test/java/com/rarchives/ripme/tst/ui/RipStatusMessageTest.java index e5fe8b43..fbd1c604 100644 --- a/src/test/java/com/rarchives/ripme/tst/ui/RipStatusMessageTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ui/RipStatusMessageTest.java @@ -3,9 +3,11 @@ package com.rarchives.ripme.tst.ui; import com.rarchives.ripme.ui.RipStatusMessage; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; public class RipStatusMessageTest { + @Test public void testConstructor() { RipStatusMessage.STATUS loadingResource = RipStatusMessage.STATUS.LOADING_RESOURCE; String path = "path/to/file"; From 9bbbf887a29e7c15bd33e207a832831f4adf6763 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 7 Mar 2021 02:46:24 +0100 Subject: [PATCH 145/512] reference local repo in readme --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7514e697..ce27bae0 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# RipMe [![Licensed under the MIT License](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/RipMeApp/ripme/blob/master/LICENSE.txt) [![Join the chat at https://gitter.im/RipMeApp/Lobby](https://badges.gitter.im/RipMeApp/Lobby.svg)](https://gitter.im/RipMeApp/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Subreddit](https://img.shields.io/badge/discuss-on%20reddit-blue.svg)](https://www.reddit.com/r/ripme/) - -[![Build Status](https://travis-ci.org/RipMeApp/ripme.svg?branch=master)](https://travis-ci.org/RipMeApp/ripme) +# RipMe +[![Licensed under the MIT License](https://img.shields.io/badge/License-MIT-blue.svg)](/LICENSE.txt) +[![Join the chat at https://gitter.im/RipMeApp/Lobby](https://badges.gitter.im/RipMeApp/Lobby.svg)](https://gitter.im/RipMeApp/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Subreddit](https://img.shields.io/badge/discuss-on%20reddit-blue.svg)](https://www.reddit.com/r/ripme/) +[![example workflow](/../../../actions/workflows/gradle.yml/badge.svg)](/../../../actions/workflows/gradle.yml) [![Coverage Status](https://coveralls.io/repos/github/RipMeApp/ripme/badge.svg?branch=master)](https://coveralls.io/github/RipMeApp/ripme?branch=master) -# Contribute - RipMe is maintained with ♥️ and in our limited free time by **[@MetaPrime](https://github.com/metaprime)**, **[@cyian-1756](https://github.com/cyian-1756)** and **[@kevin51jiang](https://github.com/kevin51jiang)**. If you'd like to contribute but aren't good with code, help keep us happy with a small contribution! [![Tip with PayPal](https://img.shields.io/badge/PayPal-Buy_us...-lightgrey.svg)](https://www.paypal.me/ripmeapp) @@ -39,7 +39,7 @@ brew install --cask ripme && xattr -d com.apple.quarantine /Applications/ripme.j ## Changelog -[Changelog](https://github.com/ripmeapp/ripme/blob/master/ripme.json) **(ripme.json)** +[Changelog](/ripme.json) **(ripme.json)** # Features From 4acda1585de6e378a6694074f8b5bf006c0f38d1 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 14 Mar 2021 09:11:47 +0100 Subject: [PATCH 146/512] hetaicafe flaky --- .../ripme/tst/ripper/rippers/HentaicafeRipperTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java index 9c922260..555c2662 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java @@ -4,16 +4,19 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.HentaiCafeRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class HentaicafeRipperTest extends RippersTest { @Test + @Tag("flaky") public void testHentaiCafeAlbum() throws IOException { HentaiCafeRipper ripper = new HentaiCafeRipper(new URL("https://hentai.cafe/kikuta-the-oni-in-the-room/")); testRipper(ripper); } // This album has a line break (
) in the url. Test it to make sure ripme can handle these invalid urls @Test + @Tag("flaky") public void testAlbumWithInvalidChars() throws IOException { HentaiCafeRipper ripper = new HentaiCafeRipper(new URL("https://hentai.cafe/chobipero-club/")); testRipper(ripper); From 4c60408fea03f626f752c5c245f7bbc54acdaa0f Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 14 Mar 2021 09:27:07 +0100 Subject: [PATCH 147/512] smuttyripper flaky --- .../rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java index cb1a78c4..4085bb56 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java @@ -5,10 +5,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.SmuttyRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class SmuttyRipperTest extends RippersTest { @Test + @Tag("flaky") public void testRip() throws IOException { SmuttyRipper ripper = new SmuttyRipper(new URL("https://smutty.com/user/QUIGON/")); testRipper(ripper); From 878fa884b5be15bb4e9f3d7949ad5e5f9b9aab93 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 14 Mar 2021 10:24:51 +0100 Subject: [PATCH 148/512] github ref name into latest tag, #12 --- .github/workflows/gradle.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index fb67e7df..8fdfbebb 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -20,6 +20,10 @@ jobs: - uses: actions/checkout@v1 + - name: Set environment CI_ variables + id: ci-env + uses: FranzDiebold/github-env-vars-action@v2 + - name: Set up JDK uses: actions/setup-java@v1 with: @@ -52,9 +56,9 @@ jobs: uses: "marvinpinto/action-automatic-releases@latest" with: repo_token: "${{ secrets.GITHUB_TOKEN }}" - automatic_release_tag: "latest" + automatic_release_tag: "latest-${{ env.CI_REF_NAME_SLUG }}" prerelease: true - title: "development build" + title: "development build ${{ env.CI_REF_NAME }}" files: | build/libs/*.jar From 4dd23d3a82ca29032594a2245a9a2610ffa1b17b Mon Sep 17 00:00:00 2001 From: Stefan Aladzic <69090391+saladzic@users.noreply.github.com> Date: Thu, 18 Mar 2021 09:20:51 +0100 Subject: [PATCH 149/512] + Gradle build status --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ce27bae0..f0e66530 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Licensed under the MIT License](https://img.shields.io/badge/License-MIT-blue.svg)](/LICENSE.txt) [![Join the chat at https://gitter.im/RipMeApp/Lobby](https://badges.gitter.im/RipMeApp/Lobby.svg)](https://gitter.im/RipMeApp/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Subreddit](https://img.shields.io/badge/discuss-on%20reddit-blue.svg)](https://www.reddit.com/r/ripme/) -[![example workflow](/../../../actions/workflows/gradle.yml/badge.svg)](/../../../actions/workflows/gradle.yml) +![alt Badge Status](https://github.com/ripmeapp2/ripme/actions/workflows/gradle.yml/badge.svg) [![Coverage Status](https://coveralls.io/repos/github/RipMeApp/ripme/badge.svg?branch=master)](https://coveralls.io/github/RipMeApp/ripme?branch=master) RipMe is maintained with ♥️ and in our limited free time by **[@MetaPrime](https://github.com/metaprime)**, **[@cyian-1756](https://github.com/cyian-1756)** and **[@kevin51jiang](https://github.com/kevin51jiang)**. If you'd like to contribute but aren't good with code, help keep us happy with a small contribution! From d68dfd391b1056d2fd806cdca7b737e15d54612a Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 20 Mar 2021 10:01:47 +0100 Subject: [PATCH 150/512] pornhubrippertest flaky --- .../rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java index 354b4e62..22de8349 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java @@ -8,6 +8,7 @@ import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; import org.jsoup.nodes.Document; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class PornhubRipperTest extends RippersTest { @@ -28,6 +29,7 @@ public class PornhubRipperTest extends RippersTest { } @Test + @Tag("flaky") public void testGetNextPage() throws IOException { String baseURL = "https://www.pornhub.com/album/30687901"; PornhubRipper ripper = new PornhubRipper(new URL(baseURL)); From c52c34ab999d7982a7f5afc63f8912024e52b995 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 20 Mar 2021 10:10:09 +0100 Subject: [PATCH 151/512] load history should not produce NPE, ripme#1853 see: * https://github.com/RipMeApp/ripme/issues/1853 * https://stackoverflow.com/questions/20714058/file-exists-and-is-directory-but-listfiles-returns-null --- .../java/com/rarchives/ripme/ui/MainWindow.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 92b8071c..48e8d836 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1183,13 +1183,15 @@ public final class MainWindow implements Runnable, RipStatusHandler { // Guess rip history based on rip folder String[] dirs = Utils.getWorkingDirectory() .list((dir, file) -> new File(dir.getAbsolutePath() + File.separator + file).isDirectory()); - for (String dir : dirs) { - String url = RipUtils.urlFromDirectoryName(dir); - if (url != null) { - // We found one, add it to history - HistoryEntry entry = new HistoryEntry(); - entry.url = url; - HISTORY.add(entry); + if (dirs != null) { + for (String dir : dirs) { + String url = RipUtils.urlFromDirectoryName(dir); + if (url != null) { + // We found one, add it to history + HistoryEntry entry = new HistoryEntry(); + entry.url = url; + HISTORY.add(entry); + } } } } From ca1c78c4912a016a133e7a424086cfa630ac653d Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Mar 2021 08:35:55 +0200 Subject: [PATCH 152/512] AerisdiesRipperTest flaky --- .../ripme/tst/ripper/rippers/AerisdiesRipperTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java index d3166240..c4c2a7a8 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java @@ -6,16 +6,19 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.AerisdiesRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class AerisdiesRipperTest extends RippersTest { @Test + @Tag("flaky") public void testAlbum() throws IOException { AerisdiesRipper ripper = new AerisdiesRipper(new URL("http://www.aerisdies.com/html/lb/alb_1097_1.html")); testRipper(ripper); } @Test + @Tag("flaky") public void testSubAlbum() throws IOException { AerisdiesRipper ripper = new AerisdiesRipper(new URL("http://www.aerisdies.com/html/lb/alb_3692_1.html")); testRipper(ripper); From 7f1ae26d9b2dbdaf80fbee34732148634bae74db Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Mar 2021 10:36:30 +0200 Subject: [PATCH 153/512] provide tasks to unit test, and describe them gradle does not take the excludeTags and includeTags parameters on the command line with gradle-6.8.3. provide tasks for it, and describe better how to run particular unit tests. --- README.md | 46 +++++++++++++++++++++++----------------------- build.gradle.kts | 18 ++++++++++++++++++ 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index f0e66530..ed630525 100644 --- a/README.md +++ b/README.md @@ -79,40 +79,40 @@ If you're a developer, you can add your own Ripper by following the wiki guide: # Compiling & Building -The project uses [Gradle](https://gradle.org) or [Maven](http://maven.apache.org/). -Therefor both commands are given. To build the .jar file, navigate to the root -project directory and run: +The project uses [Gradle](https://gradle.org). To build the .jar file, +navigate to the root project directory and run at least the test you +change, e.g. Xhamster. test execution can also excluded completely: ```bash -mvn clean compile assembly:single -mvn -B package assembly:single -Dmaven.test.skip=true -``` -```bash -./gradlew clean build +./gradlew clean build testAll --tests XhamsterRipperTest.testXhamster2Album ./gradlew clean build -x test --warning-mode all ``` -This will include all dependencies in the JAR. One can skip executing the tests -as well. +The generated JAR (java archive) in build/libs will include all +dependencies. # Running Tests -Tests can be marked as beeing slow, or flaky. Default is to run all but the flaky tests. Slow tests can be excluded to -run. slow and flaky tests can be run on its own. After building you can run tests, quoting might be necessary depending -on your shell: +Tests can be tagged as beeing slow, or flaky. The gradle build reacts to +the following combinations of tags: -```bash -mvn test -mvn test -DexcludedGroups= -Dgroups=flaky,slow -mvn test '-Dgroups=!slow' -``` +- default is to run all tests without tag. +- testAll runs all tests. +- testFlaky runs tests with tag "flaky". +- testSlow runs tests with tag "slow". +- tests can be run by test class, or single test. Use "testAll" so it does + not matter if a test is tagged or not. ```bash ./gradlew test -./gradlew test -DexcludeTags= -DincludeTags=flaky,slow -./gradlew test '-DincludeTags=!slow' +./gradlew testAll +./gradlew testFlaky +./gradlew testSlow +./gradlew testAll --tests XhamsterRipperTest +./gradlew testAll --tests XhamsterRipperTest.testXhamster2Album ``` -Please note that some tests may fail as sites change and our rippers become out of date. -Start by building and testing a released version of RipMe -and then ensure that any changes you make do not cause more tests to break. +Please note that some tests may fail as sites change and our rippers +become out of date. Start by building and testing a released version +of RipMe and then ensure that any changes you make do not cause more +tests to break. diff --git a/build.gradle.kts b/build.gradle.kts index c6ba4be2..c98ae616 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -85,6 +85,24 @@ tasks.register("testAll") { } } +tasks.register("testFlaky") { + useJUnitPlatform { + includeTags("flaky") + } +} + +tasks.register("testSlow") { + useJUnitPlatform { + includeTags("slow") + } +} + +tasks.register("testTagged") { + useJUnitPlatform { + includeTags("any()") + } +} + // make all archive tasks in the build reproducible tasks.withType().configureEach { isPreserveFileTimestamps = false From cfb0366aca54618368e4abebbc8c951dab04ab63 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Mar 2021 08:03:29 +0200 Subject: [PATCH 154/512] abstractripper avoids endless runs by checking url is seen multiple bug reports were made, like #20, RipMeApp#1854 . --- .../com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 3e3fdb18..db18b2e1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -6,10 +6,12 @@ import java.io.FileWriter; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; + import org.jsoup.nodes.Document; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; @@ -104,7 +106,15 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { LOGGER.debug("Adding items from " + this.url + " to queue"); } + List doclocation = new ArrayList<>(); while (doc != null) { + + // catch if we saw a doc location already, save the ones seen in a list + if (doclocation.contains(doc.location())) { + break; + } + doclocation.add(doc.location()); + if (alreadyDownloadedUrls >= Utils.getConfigInteger("history.end_rip_after_already_seen", 1000000000) && !isThisATest()) { sendUpdate(STATUS.DOWNLOAD_COMPLETE_HISTORY, "Already seen the last " + alreadyDownloadedUrls + " images ending rip"); break; From 9518b61b0f38211167e01606e2e2b05b222cabea Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Mar 2021 10:45:34 +0200 Subject: [PATCH 155/512] java8 syntax update of AbstractRipper --- .../ripme/ripper/AbstractHTMLRipper.java | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index db18b2e1..81ef87cd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -24,9 +24,9 @@ import com.rarchives.ripme.ui.RipStatusMessage; */ public abstract class AbstractHTMLRipper extends AbstractRipper { - private Map itemsPending = Collections.synchronizedMap(new HashMap()); - private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); - private Map itemsErrored = Collections.synchronizedMap(new HashMap()); + private final Map itemsPending = Collections.synchronizedMap(new HashMap<>()); + private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); + private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); protected AbstractHTMLRipper(URL url) throws IOException { super(url); @@ -205,7 +205,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { */ private String fileNameFromURL(URL url) { String saveAs = url.toExternalForm(); - if (saveAs.substring(saveAs.length() - 1) == "/") { saveAs = saveAs.substring(0,saveAs.length() - 1) ;} + if (saveAs.substring(saveAs.length() - 1).equals("/")) { saveAs = saveAs.substring(0,saveAs.length() - 1) ;} saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1); if (saveAs.indexOf('?') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('?')); } if (saveAs.indexOf('#') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('#')); } @@ -291,16 +291,16 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { } @Override - /** - * Returns total amount of files attempted. + /* + Returns total amount of files attempted. */ public int getCount() { return itemsCompleted.size() + itemsErrored.size(); } @Override - /** - * Queues multiple URLs of single images to download from a single Album URL + /* + Queues multiple URLs of single images to download from a single Album URL */ public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { // Only download one file if this is a test. @@ -362,8 +362,8 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { } @Override - /** - * Cleans up & tells user about successful download + /* + Cleans up & tells user about successful download */ public void downloadCompleted(URL url, File saveAs) { if (observer == null) { @@ -398,9 +398,9 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { } @Override - /** - * Tells user that a single file in the album they wish to download has - * already been downloaded in the past. + /* + Tells user that a single file in the album they wish to download has + already been downloaded in the past. */ public void downloadExists(URL url, File file) { if (observer == null) { @@ -476,13 +476,12 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { */ @Override public String getStatusText() { - StringBuilder sb = new StringBuilder(); - sb.append(getCompletionPercentage()) - .append("% ") - .append("- Pending: " ).append(itemsPending.size()) - .append(", Completed: ").append(itemsCompleted.size()) - .append(", Errored: " ).append(itemsErrored.size()); - return sb.toString(); + String sb = getCompletionPercentage() + + "% " + + "- Pending: " + itemsPending.size() + + ", Completed: " + itemsCompleted.size() + + ", Errored: " + itemsErrored.size(); + return sb; } From 4702160c75f5e9051a139dce8bdc91d8cca52262 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 13 Feb 2021 11:26:19 +0100 Subject: [PATCH 156/512] gradle, add log4j2 --- build.gradle.kts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index c98ae616..049662e9 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -11,15 +11,17 @@ repositories { } dependencies { + implementation("com.lmax:disruptor:3.4.2") implementation("org.java-websocket:Java-WebSocket:1.5.1") implementation("org.jsoup:jsoup:1.8.1") implementation("org.json:json:20190722") implementation("commons-configuration:commons-configuration:1.7") - implementation("log4j:log4j:1.2.17") implementation("commons-cli:commons-cli:1.2") implementation("commons-io:commons-io:1.3.2") implementation("org.apache.httpcomponents:httpclient:4.3.6") implementation("org.apache.httpcomponents:httpmime:4.3.3") + implementation("org.apache.logging.log4j:log4j-api:2.14.1") + implementation("org.apache.logging.log4j:log4j-core:2.14.1") implementation("org.graalvm.js:js:20.1.0") testImplementation(enforcedPlatform("org.junit:junit-bom:5.6.2")) testImplementation("org.junit.jupiter:junit-jupiter") From 684f17af8fa0f076a9d91e6a4e55b6a6dade5900 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 2 Apr 2021 20:30:46 +0200 Subject: [PATCH 157/512] test classes log4j --> log4j2 --- .../com/rarchives/ripme/tst/UtilsTest.java | 10 ++++++++++ .../ripme/tst/ripper/rippers/RippersTest.java | 20 +++++++++++-------- .../ripme/tst/ui/LabelsBundlesTest.java | 5 +++-- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/UtilsTest.java b/src/test/java/com/rarchives/ripme/tst/UtilsTest.java index d87eca55..3a8a8668 100644 --- a/src/test/java/com/rarchives/ripme/tst/UtilsTest.java +++ b/src/test/java/com/rarchives/ripme/tst/UtilsTest.java @@ -6,11 +6,21 @@ import java.util.Arrays; import com.rarchives.ripme.utils.Utils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class UtilsTest { + private final Logger LOGGER = LogManager.getLogger(UtilsTest.class); + + @Test + public void testConfigureLogger() { + Utils.configureLogger(); + LOGGER.warn("this is a warning messaage."); + } + public void testGetEXTFromMagic() { Assertions.assertEquals("jpeg", Utils.getEXTFromMagic(new byte[] { -1, -40, -1, -37, 0, 0, 0, 0 })); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java index c09b8018..b4187748 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java @@ -6,9 +6,12 @@ import java.util.List; import com.rarchives.ripme.ripper.rippers.ChanRipper; -import org.apache.log4j.ConsoleAppender; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; import org.junit.jupiter.api.Assertions; import com.rarchives.ripme.ripper.AbstractRipper; @@ -19,12 +22,16 @@ import com.rarchives.ripme.utils.Utils; */ public class RippersTest { - private final Logger logger = Logger.getLogger(RippersTest.class); + private final Logger logger = LogManager.getLogger(RippersTest.class); void testRipper(AbstractRipper ripper) { try { // Turn on Debug logging - ((ConsoleAppender) Logger.getRootLogger().getAppender("stdout")).setThreshold(Level.DEBUG); + LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + Configuration config = ctx.getConfiguration(); + LoggerConfig loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME); + loggerConfig.setLevel(Level.DEBUG); + ctx.updateLoggers(); // This causes all Loggers to refetch information from their LoggerConfig. // Decrease timeout Utils.setConfigInteger("page.timeout", 20 * 1000); @@ -56,9 +63,6 @@ public class RippersTest { // that we found links to it void testChanRipper(ChanRipper ripper) { try { - // Turn on Debug logging - ((ConsoleAppender) Logger.getRootLogger().getAppender("stdout")).setThreshold(Level.DEBUG); - // Decrease timeout Utils.setConfigInteger("page.timeout", 20 * 1000); diff --git a/src/test/java/com/rarchives/ripme/tst/ui/LabelsBundlesTest.java b/src/test/java/com/rarchives/ripme/tst/ui/LabelsBundlesTest.java index 77a590ce..d35ff49e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ui/LabelsBundlesTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ui/LabelsBundlesTest.java @@ -10,11 +10,12 @@ import java.util.Set; import com.rarchives.ripme.utils.Utils; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.Test; public class LabelsBundlesTest { - private Logger logger = Logger.getLogger(Utils.class); + private Logger logger = LogManager.getLogger(Utils.class); private static final String DEFAULT_LANG = "en_US"; @Test From f7bff9e8493467b48e989a87b6d2847cb45d3562 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 23 Jan 2021 19:58:18 +0100 Subject: [PATCH 158/512] mainwindow, configure, log4j --> log4j2 --- pom.xml | 11 ++-- .../ripme/ripper/AbstractRipper.java | 36 +++++++------ .../com/rarchives/ripme/ui/MainWindow.java | 42 +++++++-------- .../java/com/rarchives/ripme/utils/Utils.java | 51 ++++++++++++------- 4 files changed, 81 insertions(+), 59 deletions(-) diff --git a/pom.xml b/pom.xml index ccfa46a9..88e13875 100644 --- a/pom.xml +++ b/pom.xml @@ -59,9 +59,14 @@ 1.7 - log4j - log4j - 1.2.17 + org.apache.logging.log4j + log4j-api + 2.14.0 + + + org.apache.logging.log4j + log4j-core + 2.14.0 commons-cli diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 3653b9f0..132658b5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -14,8 +14,9 @@ import java.util.List; import java.util.Map; import java.util.Observable; import java.util.Scanner; -import org.apache.log4j.FileAppender; -import org.apache.log4j.Logger; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.HttpStatusException; import com.rarchives.ripme.App; import com.rarchives.ripme.ui.RipStatusComplete; @@ -28,7 +29,7 @@ public abstract class AbstractRipper extends Observable implements RipperInterface, Runnable { - protected static final Logger LOGGER = Logger.getLogger(AbstractRipper.class); + protected static final Logger LOGGER = LogManager.getLogger(AbstractRipper.class); private final String URLHistoryFile = Utils.getURLHistoryFile(); public static final String USER_AGENT = @@ -177,12 +178,15 @@ public abstract class AbstractRipper */ public void setup() throws IOException { setWorkingDir(this.url); - Logger rootLogger = Logger.getRootLogger(); - FileAppender fa = (FileAppender) rootLogger.getAppender("FILE"); - if (fa != null) { - fa.setFile(this.workingDir + File.separator + "log.txt"); - fa.activateOptions(); - } + // we do not care if the rollingfileappender is active, just change the logfile in case + // TODO this does not work - not even with + // .withFileName("${sys:logFilename}") + // in Utils.java, RollingFileAppender. +// System.setProperty("logFilename", this.workingDir + "/log.txt"); +// LOGGER.debug("Changing log file to '{}/log.txt'", this.workingDir); +// LoggerContext ctx = (LoggerContext) LogManager.getContext(false); +// ctx.reconfigure(); +// ctx.updateLoggers(); this.threadPool = new DownloadThreadPool(); } @@ -482,13 +486,13 @@ public abstract class AbstractRipper RipStatusMessage msg = new RipStatusMessage(STATUS.RIP_COMPLETE, rsc); observer.update(this, msg); - Logger rootLogger = Logger.getRootLogger(); - FileAppender fa = (FileAppender) rootLogger.getAppender("FILE"); - if (fa != null) { - LOGGER.debug("Changing log file back to 'ripme.log'"); - fa.setFile("ripme.log"); - fa.activateOptions(); - } + // we do not care if the rollingfileappender is active, just change the logfile in case + // TODO - does not work. +// System.setProperty("logFilename", "ripme.log"); +// LOGGER.debug("Changing log file back to 'ripme.log'"); +// LoggerContext ctx = (LoggerContext) LogManager.getContext(false); +// ctx.reconfigure(); + if (Utils.getConfigBoolean("urls_only.save", false)) { String urlFile = this.workingDir + File.separator + "urls.txt"; try { diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 48e8d836..935e48d9 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -49,14 +49,15 @@ import javax.swing.text.SimpleAttributeSet; import javax.swing.text.StyleConstants; import javax.swing.text.StyledDocument; -import org.apache.log4j.ConsoleAppender; -import org.apache.log4j.FileAppender; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; - import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; import javax.swing.UnsupportedLookAndFeelException; @@ -65,7 +66,7 @@ import javax.swing.UnsupportedLookAndFeelException; */ public final class MainWindow implements Runnable, RipStatusHandler { - private static final Logger LOGGER = Logger.getLogger(MainWindow.class); + private static final Logger LOGGER = LogManager.getLogger(MainWindow.class); private boolean isRipping = false; // Flag to indicate if we're ripping something @@ -993,16 +994,11 @@ public final class MainWindow implements Runnable, RipStatusHandler { newLevel = Level.ERROR; break; } - Logger.getRootLogger().setLevel(newLevel); - LOGGER.setLevel(newLevel); - ConsoleAppender ca = (ConsoleAppender) Logger.getRootLogger().getAppender("stdout"); - if (ca != null) { - ca.setThreshold(newLevel); - } - FileAppender fa = (FileAppender) Logger.getRootLogger().getAppender("FILE"); - if (fa != null) { - fa.setThreshold(newLevel); - } + LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + Configuration config = ctx.getConfiguration(); + LoggerConfig loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME); + loggerConfig.setLevel(newLevel); + ctx.updateLoggers(); // This causes all Loggers to refetch information from their LoggerConfig. } private void setupTrayIcon() { @@ -1382,34 +1378,34 @@ public final class MainWindow implements Runnable, RipStatusHandler { switch (msg.getStatus()) { case LOADING_RESOURCE: case DOWNLOAD_STARTED: - if (LOGGER.isEnabledFor(Level.INFO)) { + if (LOGGER.isEnabled(Level.INFO)) { appendLog("Downloading " + msg.getObject(), Color.BLACK); } break; case DOWNLOAD_COMPLETE: - if (LOGGER.isEnabledFor(Level.INFO)) { + if (LOGGER.isEnabled(Level.INFO)) { appendLog("Downloaded " + msg.getObject(), Color.GREEN); } break; case DOWNLOAD_COMPLETE_HISTORY: - if (LOGGER.isEnabledFor(Level.INFO)) { + if (LOGGER.isEnabled(Level.INFO)) { appendLog("" + msg.getObject(), Color.GREEN); } break; case DOWNLOAD_ERRORED: - if (LOGGER.isEnabledFor(Level.ERROR)) { + if (LOGGER.isEnabled(Level.ERROR)) { appendLog((String) msg.getObject(), Color.RED); } break; case DOWNLOAD_WARN: - if (LOGGER.isEnabledFor(Level.WARN)) { + if (LOGGER.isEnabled(Level.WARN)) { appendLog((String) msg.getObject(), Color.ORANGE); } break; case RIP_ERRORED: - if (LOGGER.isEnabledFor(Level.ERROR)) { + if (LOGGER.isEnabled(Level.ERROR)) { appendLog((String) msg.getObject(), Color.RED); } stopButton.setEnabled(false); @@ -1511,7 +1507,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { // Update total bytes break; case NO_ALBUM_OR_USER: - if (LOGGER.isEnabledFor(Level.ERROR)) { + if (LOGGER.isEnabled(Level.ERROR)) { appendLog((String) msg.getObject(), Color.RED); } stopButton.setEnabled(false); diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 7f9d99aa..b1be308d 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -3,7 +3,6 @@ package com.rarchives.ripme.utils; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.lang.reflect.Constructor; import java.net.URI; @@ -40,9 +39,15 @@ import com.rarchives.ripme.ripper.AbstractRipper; import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; -import org.apache.log4j.PropertyConfigurator; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.RollingFileAppender; +import org.apache.logging.log4j.core.appender.rolling.DefaultRolloverStrategy; +import org.apache.logging.log4j.core.appender.rolling.SizeBasedTriggeringPolicy; +import org.apache.logging.log4j.core.appender.rolling.TriggeringPolicy; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; /** * Common utility functions used in various places throughout the project. @@ -54,7 +59,7 @@ public class Utils { private static final String RIP_DIRECTORY = "rips"; private static final String CONFIG_FILE = "rip.properties"; private static final String OS = System.getProperty("os.name").toLowerCase(); - private static final Logger LOGGER = Logger.getLogger(Utils.class); + private static final Logger LOGGER = LogManager.getLogger(Utils.class); private static final int SHORTENED_PATH_LENGTH = 12; private static PropertiesConfiguration config; @@ -602,20 +607,32 @@ public class Utils { * Configures root logger, either for FILE output or just console. */ public static void configureLogger() { - LogManager.shutdown(); - String logFile = getConfigBoolean("log.save", false) ? "log4j.file.properties" : "log4j.properties"; - try (InputStream stream = Utils.class.getClassLoader().getResourceAsStream(logFile)) { - if (stream == null) { - PropertyConfigurator.configure("src/main/resources/" + logFile); - } else { - PropertyConfigurator.configure(stream); + LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + Configuration config = ctx.getConfiguration(); + LoggerConfig loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME); + + // write to ripme.log file if checked in GUI + boolean logSave = getConfigBoolean("log.save", false); + if (logSave) { + LOGGER.debug("add rolling appender ripmelog"); + TriggeringPolicy tp = SizeBasedTriggeringPolicy.createPolicy("20M"); + DefaultRolloverStrategy rs = DefaultRolloverStrategy.newBuilder().withMax("2").build(); + RollingFileAppender rolling = RollingFileAppender.newBuilder() + .setName("ripmelog") + .withFileName("ripme.log") + .withFilePattern("%d{yyyy-MM-dd HH:mm:ss} %p %m%n") + .withPolicy(tp) + .withStrategy(rs) + .build(); + loggerConfig.addAppender(rolling, null, null); + } else { + LOGGER.debug("remove rolling appender ripmelog"); + if (config.getAppender("ripmelog") != null) { + config.getAppender("ripmelog").stop(); } - - LOGGER.info("Loaded " + logFile); - } catch (IOException e) { - LOGGER.error(e.getMessage(), e); + loggerConfig.removeAppender("ripmelog"); } - + ctx.updateLoggers(); // This causes all Loggers to refetch information from their LoggerConfig. } /** From c0cd3c61349648e01d3dbc47a4c90d9755de1204 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 2 Apr 2021 20:38:38 +0200 Subject: [PATCH 159/512] rippers log4j --> log4j2 --- src/main/java/com/rarchives/ripme/App.java | 5 +++-- .../java/com/rarchives/ripme/ripper/DownloadFileThread.java | 5 +++-- .../java/com/rarchives/ripme/ripper/DownloadThreadPool.java | 6 +++--- .../com/rarchives/ripme/ripper/DownloadVideoThread.java | 6 +++--- .../com/rarchives/ripme/ripper/rippers/BooruRipper.java | 6 ++++-- .../java/com/rarchives/ripme/ripper/rippers/E621Ripper.java | 6 ++++-- .../com/rarchives/ripme/ripper/rippers/MangadexRipper.java | 1 - .../com/rarchives/ripme/ripper/rippers/PahealRipper.java | 6 ++++-- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 5 +++-- src/main/java/com/rarchives/ripme/utils/Http.java | 5 +++-- src/main/java/com/rarchives/ripme/utils/RipUtils.java | 5 +++-- 11 files changed, 33 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index 73e0bb66..2c1a0bd9 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -24,7 +24,6 @@ import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.lang.SystemUtils; -import org.apache.log4j.Logger; import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.ui.History; @@ -34,6 +33,8 @@ import com.rarchives.ripme.ui.UpdateUtils; import com.rarchives.ripme.utils.Proxy; import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /** * Entry point to application. @@ -44,7 +45,7 @@ import com.rarchives.ripme.utils.Utils; */ public class App { - public static final Logger logger = Logger.getLogger(App.class); + public static final Logger logger = LogManager.getLogger(App.class); public static String stringToAppendToFoldername = null; private static final History HISTORY = new History(); diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index c05fe0f9..98121ed1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -13,7 +13,8 @@ import java.util.ResourceBundle; import javax.net.ssl.HttpsURLConnection; import com.rarchives.ripme.ui.MainWindow; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.HttpStatusException; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; @@ -24,7 +25,7 @@ import com.rarchives.ripme.utils.Utils; * and other goodies. */ class DownloadFileThread extends Thread { - private static final Logger logger = Logger.getLogger(DownloadFileThread.class); + private static final Logger logger = LogManager.getLogger(DownloadFileThread.class); private String referrer = ""; private Map cookies = new HashMap<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java index a811c98a..e3f9e79c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java @@ -4,16 +4,16 @@ import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import org.apache.log4j.Logger; - import com.rarchives.ripme.utils.Utils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /** * Simple wrapper around a FixedThreadPool. */ public class DownloadThreadPool { - private static final Logger logger = Logger.getLogger(DownloadThreadPool.class); + private static final Logger logger = LogManager.getLogger(DownloadThreadPool.class); private ThreadPoolExecutor threadPool = null; public DownloadThreadPool() { diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index ef55e54e..f8b4b087 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -11,10 +11,10 @@ import java.net.URL; import javax.net.ssl.HttpsURLConnection; -import org.apache.log4j.Logger; - import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /** * Thread for downloading files. @@ -22,7 +22,7 @@ import com.rarchives.ripme.utils.Utils; */ class DownloadVideoThread extends Thread { - private static final Logger logger = Logger.getLogger(DownloadVideoThread.class); + private static final Logger logger = LogManager.getLogger(DownloadVideoThread.class); private URL url; private File saveAs; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/BooruRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/BooruRipper.java index 7d6b17a6..974a0061 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/BooruRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/BooruRipper.java @@ -12,12 +12,14 @@ import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.log4j.Logger; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; public class BooruRipper extends AbstractHTMLRipper { - private static final Logger logger = Logger.getLogger(BooruRipper.class); + private static final Logger logger = LogManager.getLogger(BooruRipper.class); private static Pattern gidPattern = null; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java index bac6b51f..21e0f866 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java @@ -17,7 +17,9 @@ import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.log4j.Logger; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -25,7 +27,7 @@ import org.jsoup.select.Elements; public class E621Ripper extends AbstractHTMLRipper { - private static final Logger logger = Logger.getLogger(E621Ripper.class); + private static final Logger logger = LogManager.getLogger(E621Ripper.class); private static Pattern gidPattern = null; private static Pattern gidPattern2 = null; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java index ea8c4530..cfe2e53f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java @@ -5,7 +5,6 @@ import com.rarchives.ripme.ui.History; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; -import org.apache.log4j.Logger; import org.json.JSONArray; import org.json.JSONObject; import org.jsoup.Connection; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java index d2421f37..288ca94e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java @@ -15,13 +15,15 @@ import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.log4j.Logger; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; public class PahealRipper extends AbstractHTMLRipper { - private static final Logger logger = Logger.getLogger(PahealRipper.class); + private static final Logger logger = LogManager.getLogger(PahealRipper.class); private static Map cookies = null; private static Pattern gidPattern = null; diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index d2379940..33a6fb75 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -11,7 +11,8 @@ import javax.swing.JLabel; import javax.swing.JOptionPane; import javax.swing.JScrollPane; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.json.JSONArray; import org.json.JSONObject; import org.jsoup.Connection.Response; @@ -22,7 +23,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { - private static final Logger logger = Logger.getLogger(UpdateUtils.class); + private static final Logger logger = LogManager.getLogger(UpdateUtils.class); // do not update the default version without adjusting the unit test. the real version comes from METAINF.MF private static final String DEFAULT_VERSION = "1.7.94-10-b6345398"; private static final String REPO_NAME = "ripmeapp/ripme"; diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index fb90bbd8..71107e87 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -8,7 +8,8 @@ import java.util.Map; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.json.JSONObject; import org.jsoup.Connection; import org.jsoup.Connection.Method; @@ -28,7 +29,7 @@ import com.rarchives.ripme.ripper.AbstractRipper; public class Http { private static final int TIMEOUT = Utils.getConfigInteger("page.timeout", 5 * 1000); - private static final Logger logger = Logger.getLogger(Http.class); + private static final Logger logger = LogManager.getLogger(Http.class); private int retries; private String url; diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 503a8165..b001ff01 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -16,14 +16,15 @@ import com.rarchives.ripme.ripper.rippers.VidbleRipper; import com.rarchives.ripme.ripper.rippers.GfycatRipper; import com.rarchives.ripme.ripper.rippers.SoundgasmRipper; import org.apache.commons.lang.math.NumberUtils; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; public class RipUtils { - private static final Logger logger = Logger.getLogger(RipUtils.class); + private static final Logger logger = LogManager.getLogger(RipUtils.class); public static List getFilesFromURL(URL url) { List result = new ArrayList<>(); From 2cdf79babc83573f247493b448f0ee2ffc617c1d Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 3 Apr 2021 03:47:19 +0200 Subject: [PATCH 160/512] log4j properties --> log4j2-example.xml --- src/main/resources/log4j.properties | 10 --------- src/main/resources/log4j2-example.xml | 30 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 10 deletions(-) delete mode 100644 src/main/resources/log4j.properties create mode 100644 src/main/resources/log4j2-example.xml diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties deleted file mode 100644 index 409dd303..00000000 --- a/src/main/resources/log4j.properties +++ /dev/null @@ -1,10 +0,0 @@ - -# define the console appender -log4j.appender.stdout = org.apache.log4j.ConsoleAppender -log4j.appender.stdout.Target = System.out -log4j.appender.stdout.Threshold = info -log4j.appender.stdout.layout = org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern = %m%n - -# now map our console appender as a root logger, means all log messages will go to this appender -log4j.rootLogger = debug, stdout \ No newline at end of file diff --git a/src/main/resources/log4j2-example.xml b/src/main/resources/log4j2-example.xml new file mode 100644 index 00000000..dbc0888c --- /dev/null +++ b/src/main/resources/log4j2-example.xml @@ -0,0 +1,30 @@ + + + + # Console appender + + # Pattern of log message for console appender + + + + # Rolling appender + + + %d{yyyy-MM-dd HH:mm:ss} %p %m%n + + + + + + + + + + + + + + + From d6941f429e9c09f831de01b0a9bac2d398fecda7 Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 6 Apr 2021 20:33:20 +0200 Subject: [PATCH 161/512] utf-8 in title --- .../com/rarchives/ripme/ripper/rippers/EightmusesRipper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java index 22968216..82150b89 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java @@ -186,9 +186,10 @@ public class EightmusesRipper extends AbstractHTMLRipper { private String deobfuscateChar(char c) { if ((int) c == 32) { return fromCharCode(32); + } else if ((int) c > 120){ + return fromCharCode((int)c); } return fromCharCode(33 + (c + 14) % 94); - } private static String fromCharCode(int... codePoints) { From 886d345dd7d2f030232c6536818ed334017a2ce5 Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 6 Apr 2021 20:36:31 +0200 Subject: [PATCH 162/512] redgifrippertest flaky --- .../rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java index 8b45594d..ed71128d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java @@ -36,6 +36,7 @@ public class RedgifsRipperTest extends RippersTest { * @throws IOException */ @Test + @Tag("flaky") public void testRedgifsProfile() throws IOException { RedgifsRipper ripper = new RedgifsRipper(new URL("https://redgifs.com/users/margo_monty")); testRipper(ripper); From aae3f5c38caf04e21c613ce8b9f23d65fcde3c55 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 11 Apr 2021 18:34:47 +0200 Subject: [PATCH 163/512] remove travis, eclipse files --- .project | 23 ----------------------- .travis.yml | 12 ------------ workspace.code-workspace | 16 ---------------- 3 files changed, 51 deletions(-) delete mode 100644 .project delete mode 100644 .travis.yml delete mode 100644 workspace.code-workspace diff --git a/.project b/.project deleted file mode 100644 index 89407457..00000000 --- a/.project +++ /dev/null @@ -1,23 +0,0 @@ - - - ripme - - - - - - org.eclipse.jdt.core.javabuilder - - - - - org.eclipse.m2e.core.maven2Builder - - - - - - org.eclipse.jdt.core.javanature - org.eclipse.m2e.core.maven2Nature - - diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0fd1b17f..00000000 --- a/.travis.yml +++ /dev/null @@ -1,12 +0,0 @@ -language: java - -matrix: - include: - - jdk: openjdk9 - before_install: - - rm "${JAVA_HOME}/lib/security/cacerts" - - ln -s /etc/ssl/certs/java/cacerts "${JAVA_HOME}/lib/security/cacerts" - - jdk: openjdk8 - -after_success: - - mvn clean test jacoco:report coveralls:report diff --git a/workspace.code-workspace b/workspace.code-workspace deleted file mode 100644 index 95b80106..00000000 --- a/workspace.code-workspace +++ /dev/null @@ -1,16 +0,0 @@ -{ - "folders": [ - { - "path": "E:\\Downloads\\_Isaaku\\dev" - } - ], - "settings": { - "files.exclude": { - "**/.classpath": false, - "**/.project": true, - "**/.settings": true, - "**/.factorypath": true - }, - "java.configuration.updateBuildConfiguration": "automatic" - } -} \ No newline at end of file From c7af5c503da1eaf9bfae0aeafe6cf29bced812bb Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 11 Apr 2021 19:03:06 +0200 Subject: [PATCH 164/512] remove mvn maven build, fix #30. --- .github/workflows/maven.yml | 34 ------- build.bat | 4 +- build.sh | 5 +- deploy.bat | 2 - deploy.ps1 | 16 ---- java | 0 patch.py | 86 ------------------ pom.xml | 176 ------------------------------------ 8 files changed, 4 insertions(+), 319 deletions(-) delete mode 100644 .github/workflows/maven.yml delete mode 100644 deploy.bat delete mode 100644 deploy.ps1 delete mode 100644 java delete mode 100644 patch.py delete mode 100644 pom.xml diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml deleted file mode 100644 index 038c890f..00000000 --- a/.github/workflows/maven.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Java CI - -on: workflow_dispatch - -jobs: - build: - - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] - java: [1.8] - include: # test newest java on one os only, upload from ubuntu java8 - - os: ubuntu-latest - java: 1.15 - - os: ubuntu-latest - upload: true - - steps: - - uses: actions/checkout@v1 - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - name: Build with Maven - run: mvn -B package assembly:single --file pom.xml - - name: upload jar as asset - if: matrix.upload - uses: actions/upload-artifact@v2 - with: - name: zipped-ripme-jar - path: target/*dependencies.jar - -# vim:set ts=2 sw=2 et: diff --git a/build.bat b/build.bat index 7c2aa6c3..719662c9 100755 --- a/build.bat +++ b/build.bat @@ -1,2 +1,2 @@ -mvn clean compile assembly:single -mvn io.github.zlika:reproducible-build-maven-plugin:0.6:strip-jar \ No newline at end of file +./gradlew clean build -x test +./gradlew testAll diff --git a/build.sh b/build.sh index 2f044cde..68578762 100755 --- a/build.sh +++ b/build.sh @@ -1,4 +1,3 @@ #!/usr/bin/env bash -mvn clean compile assembly:single -# Strip the jar of any non-reproducible metadata such as timestamps -mvn io.github.zlika:reproducible-build-maven-plugin:0.6:strip-jar \ No newline at end of file +./gradlew clean build -x test +./gradlew testAll diff --git a/deploy.bat b/deploy.bat deleted file mode 100644 index 388ece44..00000000 --- a/deploy.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off -powershell -c ".\deploy.ps1 -source (Join-Path target (Get-Item -Path .\target\* -Filter *.jar)[0].Name) -dest ripme.jar" diff --git a/deploy.ps1 b/deploy.ps1 deleted file mode 100644 index 9124c241..00000000 --- a/deploy.ps1 +++ /dev/null @@ -1,16 +0,0 @@ -Param ( - [Parameter(Mandatory=$True)] - [string]$source, - [Parameter(Mandatory=$True)] - [string]$dest -) - -Copy-Item -Path $source -Destination $dest - -$sourceHash = (Get-FileHash $source -algorithm MD5).Hash -$destHash = (Get-FileHash $dest -algorithm MD5).Hash -if ($sourceHash -eq $destHash) { - Write-Output 'Deployed successfully.' -} else { - Write-Output 'Hash Mismatch: did you close ripme before deploying?' -} diff --git a/java b/java deleted file mode 100644 index e69de29b..00000000 diff --git a/patch.py b/patch.py deleted file mode 100644 index aa53755d..00000000 --- a/patch.py +++ /dev/null @@ -1,86 +0,0 @@ -import json -import subprocess -from hashlib import sha256 - -# This script will: -# - read current version -# - increment patch version -# - update version in a few places -# - insert new line in ripme.json with message -# - build ripme -# - add the hash of the latest binary to ripme.json -# - commit all changes -message = input('message: ') - -# Strip any spaces that might've been entered before the message -message.lstrip() - - -def get_ripme_json(): - with open('ripme.json') as dataFile: - ripmeJson = json.load(dataFile) - return ripmeJson - - -def update_hash(current_hash): - ripmeJson = get_ripme_json() - with open('ripme.json', 'w') as dataFile: - ripmeJson["currentHash"] = current_hash - print(ripmeJson["currentHash"]) - json.dump(ripmeJson, dataFile, indent=4) - - -def update_change_list(message): - ripmeJson = get_ripme_json() - with open('ripme.json', 'w') as dataFile: - ripmeJson["changeList"].insert(0, message) - json.dump(ripmeJson, dataFile, indent=4) - - -currentVersion = get_ripme_json()["latestVersion"] - -print('Current version ' + currentVersion) - -versionFields = currentVersion.split('.') -patchCur = int(versionFields[2]) -patchNext = patchCur + 1 -majorMinor = versionFields[:2] -majorMinor.append(str(patchNext)) -nextVersion = '.'.join(majorMinor) - -print('Updating to ' + nextVersion) - -substrExpr = 's/' + currentVersion + '/' + nextVersion + '/' -subprocess.call(['sed', '-i', '-e', substrExpr, 'src/main/java/com/rarchives/ripme/ui/UpdateUtils.java']) -subprocess.call(['git', 'grep', 'DEFAULT_VERSION.*' + nextVersion, - 'src/main/java/com/rarchives/ripme/ui/UpdateUtils.java']) - -substrExpr = 's/\\\"latestVersion\\\": \\\"' + currentVersion + '\\\"/\\\"latestVersion\\\": \\\"' + \ - nextVersion + '\\\"/' -subprocess.call(['sed', '-i', '-e', substrExpr, 'ripme.json']) -subprocess.call(['git', 'grep', 'latestVersion', 'ripme.json']) - -substrExpr = 's/' + currentVersion + '/' + nextVersion + '/' -subprocess.call(['sed', '-i', '-e', substrExpr, 'pom.xml']) -subprocess.call(['git', 'grep', '' + nextVersion + '', 'pom.xml']) - -commitMessage = nextVersion + ': ' + message - -update_change_list(commitMessage) - - -print("Building ripme") -subprocess.call(["mvn", "clean", "compile", "assembly:single"]) -print("Stripping jar") -subprocess.call(["mvn", "io.github.zlika:reproducible-build-maven-plugin:0.6:strip-jar"]) -print("Hashing .jar file") -openedFile = open("./target/ripme-{}-jar-with-dependencies.jar".format(nextVersion), "rb") -readFile = openedFile.read() -file_hash = sha256(readFile).hexdigest() -print("Hash is: {}".format(file_hash)) -print("Updating hash") -update_hash(file_hash) -subprocess.call(['git', 'add', '-u']) -subprocess.call(['git', 'commit', '-m', commitMessage]) -subprocess.call(['git', 'tag', nextVersion]) -print("Remember to run `git push origin master` before release.py") diff --git a/pom.xml b/pom.xml deleted file mode 100644 index 88e13875..00000000 --- a/pom.xml +++ /dev/null @@ -1,176 +0,0 @@ - - 4.0.0 - com.rarchives.ripme - ripme - jar - 1.7.94 - ripme - http://rip.rarchives.com - - flaky - UTF-8 - - - - - org.junit - junit-bom - 5.6.2 - pom - import - - - - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - org.junit.vintage - junit-vintage-engine - test - - - - org.jsoup - jsoup - 1.8.1 - - - org.graalvm.js - js - 20.1.0 - - - org.json - json - 20190722 - - - commons-configuration - commons-configuration - 1.7 - - - org.apache.logging.log4j - log4j-api - 2.14.0 - - - org.apache.logging.log4j - log4j-core - 2.14.0 - - - commons-cli - commons-cli - 1.2 - - - commons-io - commons-io - 1.3.2 - - - org.apache.httpcomponents - httpclient - 4.3.6 - - - org.apache.httpcomponents - httpmime - 4.3.3 - - - org.java-websocket - Java-WebSocket - 1.5.1 - - - - - - org.apache.maven.plugins - maven-site-plugin - 3.7.1 - - - io.github.zlika - reproducible-build-maven-plugin - 0.6 - - - maven-assembly-plugin - - - - com.rarchives.ripme.App - true - true - - - ./config - - - - jar-with-dependencies - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - 1.8 - 1.8 - - - - org.eluder.coveralls - coveralls-maven-plugin - 4.3.0 - - - - org.jacoco - jacoco-maven-plugin - 0.8.6 - - - prepare-agent - - prepare-agent - - - - - - maven-surefire-plugin - 3.0.0-M5 - - ${excludedGroups} - - - - - - - - org.apache.maven.plugins - maven-surefire-report-plugin - 3.0.0-M5 - - false - - - - - From f4b070890b4c722cacd7a052fde7c1bd3b5495e9 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 11 Apr 2021 19:10:55 +0200 Subject: [PATCH 165/512] remove PorncomixDotOneRipper, #24 --- .../ripper/rippers/PorncomixDotOneRipper.java | 68 ------------------- .../rippers/PorncomixDotOneRipperTest.java | 17 ----- 2 files changed, 85 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixDotOneRipper.java delete mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixDotOneRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixDotOneRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixDotOneRipper.java deleted file mode 100644 index c1e7fac7..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixDotOneRipper.java +++ /dev/null @@ -1,68 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; - -public class PorncomixDotOneRipper extends AbstractHTMLRipper { - - public PorncomixDotOneRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "porncomix"; - } - - @Override - public String getDomain() { - return "porncomix.one"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("https?://www.porncomix.one/gallery/([a-zA-Z0-9_\\-]*)/?$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - throw new MalformedURLException("Expected proncomix URL format: " + - "porncomix.one/gallery/comic - got " + url + " instead"); - } - - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - - @Override - public List getURLsFromPage(Document doc) { - List result = new ArrayList<>(); - // We have 2 loops here to cover all the different album types - for (Element el : doc.select(".dgwt-jg-item > a")) { - result.add(el.attr("href")); - } - for (Element el : doc.select(".unite-gallery > img")) { - result.add(el.attr("data-image")); - - } - return result; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } -} \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixDotOneRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixDotOneRipperTest.java deleted file mode 100644 index 1edf7b80..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixDotOneRipperTest.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URL; - -import com.rarchives.ripme.ripper.rippers.PorncomixDotOneRipper; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -public class PorncomixDotOneRipperTest extends RippersTest { - @Test - @Disabled("website down?") - public void testPorncomixAlbum() throws IOException { - PorncomixDotOneRipper ripper = new PorncomixDotOneRipper(new URL("https://www.porncomix.one/gallery/blacknwhite-make-america-great-again")); - testRipper(ripper); - } -} \ No newline at end of file From 431d382c5feb53949583acd66948f9a9fb7fb30c Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 11 Apr 2021 19:17:06 +0200 Subject: [PATCH 166/512] remove DrawcrowdRipper, #24 --- .../ripme/ripper/rippers/DrawcrowdRipper.java | 91 ------------------- .../ripper/rippers/DrawcrowdRipperTest.java | 19 ---- 2 files changed, 110 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/DrawcrowdRipper.java delete mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/DrawcrowdRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DrawcrowdRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DrawcrowdRipper.java deleted file mode 100644 index 521bc7c4..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DrawcrowdRipper.java +++ /dev/null @@ -1,91 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; - -public class DrawcrowdRipper extends AbstractHTMLRipper { - - public DrawcrowdRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "drawcrowd"; - } - @Override - public String getDomain() { - return "drawcrowd.com"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p; Matcher m; - - p = Pattern.compile("^.*drawcrowd.com/projects/.*$"); - m = p.matcher(url.toExternalForm()); - if (m.matches()) { - throw new MalformedURLException("Cannot rip drawcrowd.com/projects/ pages"); - } - - p = Pattern.compile("^.*drawcrowd.com/([a-zA-Z0-9\\-_]+).*$"); - m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - - throw new MalformedURLException( - "Expected drawcrowd.com gallery format: " - + "drawcrowd.com/username" - + " Got: " + url); - } - - @Override - public Document getFirstPage() throws IOException { - return Http.url(this.url).get(); - } - - @Override - public Document getNextPage(Document doc) throws IOException { - Elements loadMore = doc.select("a#load-more"); - if (loadMore.isEmpty()) { - throw new IOException("No next page found"); - } - if (!sleep(1000)) { - throw new IOException("Interrupted while waiting for next page"); - } - String nextPage = "http://drawcrowd.com" + loadMore.get(0).attr("href"); - return Http.url(nextPage).get(); - } - - @Override - public List getURLsFromPage(Document page) { - List imageURLs = new ArrayList<>(); - for (Element thumb : page.select("div.item.asset img")) { - String image = thumb.attr("src"); - image = image - .replaceAll("/medium/", "/large/") - .replaceAll("/small/", "/large/"); - imageURLs.add(image); - } - return imageURLs; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } - -} \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DrawcrowdRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DrawcrowdRipperTest.java deleted file mode 100644 index b326d365..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DrawcrowdRipperTest.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URL; - -import com.rarchives.ripme.ripper.rippers.DrawcrowdRipper; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -public class DrawcrowdRipperTest extends RippersTest { - @Test - @Disabled("https://github.com/RipMeApp/ripme/issues/304 -- Drawcrowd broken (site changed)") - public void testRip() throws IOException { - DrawcrowdRipper ripper = new DrawcrowdRipper(new URL("https://drawcrowd.com/rabbiteyes")); - testRipper(ripper); - } - -} From f587e175e35148c6f810149e5507346a0ed1c829 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 16 Apr 2021 06:41:47 +0200 Subject: [PATCH 167/512] log4j2, java-11, need "Multi-Release: true" in MANIFEST if "Multi-Release: true" is not in MANIFEST, a WARNING: sun.reflect.Reflection.getCallerClass is not supported. This will impact performance. is displayed, and DEBUG logging not turned on. see: https://stackoverflow.com/questions/53049346/is-log4j2-compatible-with-java-11 fixes #33 --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 049662e9..8ba11067 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -47,7 +47,7 @@ tasks.withType { manifest { attributes["Main-Class"] = "com.rarchives.ripme.App" attributes["Implementation-Version"] = archiveVersion - + attributes["Multi-Release"] = "true" } // To add all of the dependencies otherwise a "NoClassDefFoundError" error From 44dfdca8190b95c713f825657dc4f09eb88f4165 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 18 Apr 2021 12:59:19 +0200 Subject: [PATCH 168/512] exception when creating a directory for downloads fails. --- .../rarchives/ripme/ripper/AbstractHTMLRipper.java | 4 +++- .../rarchives/ripme/ripper/AbstractJSONRipper.java | 13 ++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 81ef87cd..86d01cf9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -455,7 +455,9 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { this.workingDir = new File(path); if (!this.workingDir.exists()) { LOGGER.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir)); - this.workingDir.mkdirs(); + if (!this.workingDir.mkdirs()) { + throw new IOException("Failed creating dir: \"" + this.workingDir + "\""); + } } LOGGER.debug("Set working directory to: " + this.workingDir); } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index d7e93fcb..6dbd8b8e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -1,5 +1,10 @@ package com.rarchives.ripme.ripper; +import com.rarchives.ripme.ui.RipStatusMessage; +import com.rarchives.ripme.ui.RipStatusMessage.STATUS; +import com.rarchives.ripme.utils.Utils; +import org.json.JSONObject; + import java.io.File; import java.io.FileWriter; import java.io.IOException; @@ -9,10 +14,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.json.JSONObject; -import com.rarchives.ripme.ui.RipStatusMessage; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; -import com.rarchives.ripme.utils.Utils; /** * Simplified ripper, designed for ripping from sites by parsing JSON. @@ -293,7 +294,9 @@ public abstract class AbstractJSONRipper extends AbstractRipper { this.workingDir = new File(path); if (!this.workingDir.exists()) { LOGGER.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir)); - this.workingDir.mkdirs(); + if (!this.workingDir.mkdirs()) { + throw new IOException("Failed creating dir: \"" + this.workingDir + "\""); + } } LOGGER.debug("Set working directory to: " + this.workingDir); } From 7d4d2ef8fa9046030d5166f434d4d3fc4c36c784 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 18 Apr 2021 13:29:47 +0200 Subject: [PATCH 169/512] check if the directory to store things is writable --- src/main/java/com/rarchives/ripme/utils/Utils.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index b1be308d..842d3e0a 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -509,7 +509,7 @@ public class Utils { * @param path - original path entered to be ripped * @return path of existing folder or the original path if not present */ - public static String getOriginalDirectory(String path) { + public static String getOriginalDirectory(String path) throws IOException { int index; if (isUnix() || isMacOS()) { @@ -524,6 +524,9 @@ public class Utils { // Get a List of all Directories and check its lowercase // if file exists return it File file = new File(path.substring(0, index)); + if (! (file.isDirectory() && file.canWrite() && file.canExecute())) { + throw new IOException("Original directory \"" + file + "\" is no directory or not writeable."); + } ArrayList names = new ArrayList<>(Arrays.asList(file.list())); for (String name : names) { From 57ad75b76eb67f2a9e7802e19d2a1f3066bbe514 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 18 Apr 2021 16:02:22 +0200 Subject: [PATCH 170/512] code cleanup --- src/main/java/com/rarchives/ripme/App.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index 2c1a0bd9..d45bc4e3 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -55,7 +55,7 @@ public class App { * * @param args Array of command line arguments. */ - public static void main(String[] args) throws MalformedURLException { + public static void main(String[] args) { CommandLine cl = getArgs(args); if (args.length > 0 && cl.hasOption('v')){ @@ -114,7 +114,7 @@ public class App { entry.dir = ripper.getWorkingDir().getAbsolutePath(); try { entry.title = ripper.getAlbumTitle(ripper.getURL()); - } catch (MalformedURLException e) { } + } catch (MalformedURLException ignored) { } HISTORY.add(entry); } } From 4743b09d5b7ca636b1e505f0ece1858ce3d37879 Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 21 Apr 2021 23:39:57 +0200 Subject: [PATCH 171/512] move zh_CN properties into resources --- src/{ => main/resources}/LabelsBundle_zh_CN.properties | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{ => main/resources}/LabelsBundle_zh_CN.properties (100%) diff --git a/src/LabelsBundle_zh_CN.properties b/src/main/resources/LabelsBundle_zh_CN.properties similarity index 100% rename from src/LabelsBundle_zh_CN.properties rename to src/main/resources/LabelsBundle_zh_CN.properties From fbf4273ec3af67fa3e43313bad81ff9413ce10f0 Mon Sep 17 00:00:00 2001 From: Tin Tin Hamans <5984296+tintinhamans@users.noreply.github.com> Date: Thu, 29 Apr 2021 11:23:44 +0200 Subject: [PATCH 172/512] move l option to before r and R --- src/main/java/com/rarchives/ripme/App.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index 73e0bb66..1952fdda 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -169,6 +169,12 @@ public class App { Utils.setConfigBoolean("errors.skip404", true); } + //Destination directory + if (cl.hasOption('l')) { + // change the default rips directory + Utils.setConfigString("rips.directory", cl.getOptionValue('l')); + } + //Re-rip all previous albums if (cl.hasOption('r')) { // Re-rip all via command-line @@ -245,12 +251,6 @@ public class App { System.exit(-1); } - //Destination directory - if (cl.hasOption('l')) { - // change the default rips directory - Utils.setConfigString("rips.directory", cl.getOptionValue('l')); - } - //Read URLs from File if (cl.hasOption('f')) { String filename = cl.getOptionValue('f'); From 6296881ffcc8d5c3d5009018fb65a45844957fb9 Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 29 Apr 2021 23:28:04 +0200 Subject: [PATCH 173/512] update commons-io to 2.7 --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 8ba11067..a6ab785f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -17,7 +17,7 @@ dependencies { implementation("org.json:json:20190722") implementation("commons-configuration:commons-configuration:1.7") implementation("commons-cli:commons-cli:1.2") - implementation("commons-io:commons-io:1.3.2") + implementation("commons-io:commons-io:2.7") implementation("org.apache.httpcomponents:httpclient:4.3.6") implementation("org.apache.httpcomponents:httpmime:4.3.3") implementation("org.apache.logging.log4j:log4j-api:2.14.1") From 4bd7f5adc29733a41d81a3a833e1a48cf7a3b586 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 16 Apr 2021 07:52:49 +0200 Subject: [PATCH 174/512] split instagram testcases in album and single --- .../tst/ripper/rippers/InstagramRipperTest.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java index 85b3b248..c7af1a16 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java @@ -3,6 +3,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.InstagramRipper; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -37,10 +38,8 @@ public class InstagramRipperTest extends RippersTest { @Test @Disabled("Ripper broken for single items") - public void testInstagramAlbums() throws IOException { + public void testInstagramSingle() throws IOException { List contentURLs = new ArrayList<>(); - // This unit test is a bit flaky - //contentURLs.add(new URL("https://www.instagram.com/Test_User/")); contentURLs.add(new URL("https://www.instagram.com/p/BaNPpaHn2zU/?hl=en")); contentURLs.add(new URL("https://www.instagram.com/p/BaNPpaHn2zU/")); for (URL url : contentURLs) { @@ -48,4 +47,14 @@ public class InstagramRipperTest extends RippersTest { testRipper(ripper); } } + + @Test + @Tag("flaky") + public void testInstagramAlbums() throws IOException { + // do not test, in case of rate limit 200/hr since 2021. see + // https://github.com/ripmeapp2/ripme/issues/32 + URL url = new URL("https://www.instagram.com/Test_User/"); + InstagramRipper ripper = new InstagramRipper(url); + testRipper(ripper); + } } From 92b4d55c57670a13c9e555bd4923ae5c7e9a1d8a Mon Sep 17 00:00:00 2001 From: Tin Tin Hamans <5984296+tintinhamans@users.noreply.github.com> Date: Thu, 29 Apr 2021 11:23:44 +0200 Subject: [PATCH 175/512] move l option to before r and R --- src/main/java/com/rarchives/ripme/App.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index d45bc4e3..b65bcbae 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -170,6 +170,12 @@ public class App { Utils.setConfigBoolean("errors.skip404", true); } + //Destination directory + if (cl.hasOption('l')) { + // change the default rips directory + Utils.setConfigString("rips.directory", cl.getOptionValue('l')); + } + //Re-rip all previous albums if (cl.hasOption('r')) { // Re-rip all via command-line @@ -246,12 +252,6 @@ public class App { System.exit(-1); } - //Destination directory - if (cl.hasOption('l')) { - // change the default rips directory - Utils.setConfigString("rips.directory", cl.getOptionValue('l')); - } - //Read URLs from File if (cl.hasOption('f')) { String filename = cl.getOptionValue('f'); From 7dc742dfcc5fc6a0b00e9cf7ba39ebd26ecd0f3d Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 1 May 2021 08:20:27 +0200 Subject: [PATCH 176/512] zizkiripper either broken or flaky --- .../ripme/tst/ripper/rippers/ZizkiRipperTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java index 7af10e55..3d21df97 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java @@ -5,20 +5,27 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.ZizkiRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class ZizkiRipperTest extends RippersTest { + + @Test + @Tag("flaky") public void testRip() throws IOException { ZizkiRipper ripper = new ZizkiRipper(new URL("http://zizki.com/dee-chorde/we-got-spirit")); testRipper(ripper); } + @Test public void testGetGID() throws IOException { URL url = new URL("http://zizki.com/dee-chorde/we-got-spirit"); ZizkiRipper ripper = new ZizkiRipper(url); Assertions.assertEquals("dee-chorde", ripper.getGID(url)); } + @Test + @Tag("flaky") public void testAlbumTitle() throws IOException { URL url = new URL("http://zizki.com/dee-chorde/we-got-spirit"); ZizkiRipper ripper = new ZizkiRipper(url); From 9328f7a346ffc0195c73b3586617c1e8957772ba Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 2 May 2021 11:17:53 +0200 Subject: [PATCH 177/512] add compose-jb to settings.gradle.kts --- settings.gradle.kts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/settings.gradle.kts b/settings.gradle.kts index 25d89451..5528f49d 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1 +1,9 @@ +pluginManagement { + repositories { + mavenLocal() + gradlePluginPortal() + // TODO: remove after new build of compose-jb is published + maven("https://maven.pkg.jetbrains.space/public/p/compose/dev") + } +} rootProject.name = "ripme" From 3ba1350f1bbea12025f29c4e5b14a102cdfd3399 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 2 May 2021 11:46:23 +0200 Subject: [PATCH 178/512] force official kotlin style, just in case in case kotlin code is committed, make sure the official style is used, see: https://kotlinlang.org/docs/code-style-migration-guide.html#in-gradle --- .gitignore | 1 - gradle.properties | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 gradle.properties diff --git a/.gitignore b/.gitignore index fb5ed210..896c43fe 100644 --- a/.gitignore +++ b/.gitignore @@ -118,7 +118,6 @@ rips/ .history ripme.jar.update *.swp -*.properties !LabelsBundle*.properties history.json *.iml diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 00000000..7fc6f1ff --- /dev/null +++ b/gradle.properties @@ -0,0 +1 @@ +kotlin.code.style=official From 8aa3b21c4e39534faee4e30a6ad973c1714d5b40 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 4 May 2021 14:08:52 -0400 Subject: [PATCH 179/512] 1.7.95: Added porncomixinfo.net; Fixed ripper for HentaiNexus; move l option to before r and R; marked some tests as flaky --- pom.xml | 2 +- ripme.json | 5 +++-- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index fb1bb42e..37136d8e 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.rarchives.ripme ripme jar - 1.7.94 + 1.7.95 ripme http://rip.rarchives.com diff --git a/ripme.json b/ripme.json index 79030d5e..dea957c0 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "currentHash": "1ebe144d0f294c4e8b482742a2588201f8b962fbcebbbd109627e1fc9e2b6d94", + "currentHash": "008201e406f401b27248277a4188f26203bb9da0170872de900125f8a6c8b558", "changeList": [ + "1.7.95: Added porncomixinfo.net; Fixed ripper for HentaiNexus; move l option to before r and R; marked some tests as flaky ", "1.7.94: Added reddit gallery support; Fixed AllporncomicRipper; Fix imagefap ripper; instagramRipper, replaced Nashorn with GraalVM.js", "1.7.93: Fixed Motherless ripper; Fixed e621 ripper; Updated pt_PT translation; Implemented redgifs Ripper; added missing translation to Korean/KR; Fixed elecx ripper; Added ripper for HentaiNexus", "1.7.92: Added read-comic.com ripper; Fix Pawoo ripper; Add ChineseSimplified language file; Fixed artstation ripper", @@ -266,5 +267,5 @@ "1.0.3: Added VK.com ripper", "1.0.1: Added auto-update functionality" ], - "latestVersion": "1.7.94" + "latestVersion": "1.7.95" } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 8c1b415e..a255496b 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -23,7 +23,7 @@ import com.rarchives.ripme.utils.Utils; public class UpdateUtils { private static final Logger logger = Logger.getLogger(UpdateUtils.class); - private static final String DEFAULT_VERSION = "1.7.94"; + private static final String DEFAULT_VERSION = "1.7.95"; private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; private static String mainFileName; From fd6f947c9e890c652f4d703d89b170dfafeaca26 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 8 May 2021 16:57:52 +0200 Subject: [PATCH 180/512] gradle-6.8.3 --> gradle-7.0 --- build.gradle.kts | 1 + gradle/wrapper/gradle-wrapper.properties | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index a6ab785f..6edf7f77 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -44,6 +44,7 @@ java { } tasks.withType { + duplicatesStrategy = DuplicatesStrategy.INCLUDE manifest { attributes["Main-Class"] = "com.rarchives.ripme.App" attributes["Implementation-Version"] = archiveVersion diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 442d9132..f371643e 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.0-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From f29a2e8ce933059e0f74306ba39632c7d9ce7a19 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 8 May 2021 16:57:52 +0200 Subject: [PATCH 181/512] gradle-6.8.3 --> gradle-7.0 --- build.gradle.kts | 1 + gradle/wrapper/gradle-wrapper.properties | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 5572205f..cc164234 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -35,6 +35,7 @@ java { } tasks.withType { + duplicatesStrategy = DuplicatesStrategy.INCLUDE manifest { attributes["Main-Class"] = "com.rarchives.ripme.App" } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 2a563242..d8442f55 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.0-bin.zip zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists +zipStorePath=wrapper/dists \ No newline at end of file From 08582707c732aec80336d885ffcb294d5d8f0bbc Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 11 Apr 2021 19:17:06 +0200 Subject: [PATCH 182/512] remove DrawcrowdRipper, #24 --- .../ripme/ripper/rippers/DrawcrowdRipper.java | 91 ------------------- .../ripper/rippers/DrawcrowdRipperTest.java | 19 ---- 2 files changed, 110 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/DrawcrowdRipper.java delete mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/DrawcrowdRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DrawcrowdRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DrawcrowdRipper.java deleted file mode 100644 index 521bc7c4..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DrawcrowdRipper.java +++ /dev/null @@ -1,91 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; - -public class DrawcrowdRipper extends AbstractHTMLRipper { - - public DrawcrowdRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "drawcrowd"; - } - @Override - public String getDomain() { - return "drawcrowd.com"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p; Matcher m; - - p = Pattern.compile("^.*drawcrowd.com/projects/.*$"); - m = p.matcher(url.toExternalForm()); - if (m.matches()) { - throw new MalformedURLException("Cannot rip drawcrowd.com/projects/ pages"); - } - - p = Pattern.compile("^.*drawcrowd.com/([a-zA-Z0-9\\-_]+).*$"); - m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - - throw new MalformedURLException( - "Expected drawcrowd.com gallery format: " - + "drawcrowd.com/username" - + " Got: " + url); - } - - @Override - public Document getFirstPage() throws IOException { - return Http.url(this.url).get(); - } - - @Override - public Document getNextPage(Document doc) throws IOException { - Elements loadMore = doc.select("a#load-more"); - if (loadMore.isEmpty()) { - throw new IOException("No next page found"); - } - if (!sleep(1000)) { - throw new IOException("Interrupted while waiting for next page"); - } - String nextPage = "http://drawcrowd.com" + loadMore.get(0).attr("href"); - return Http.url(nextPage).get(); - } - - @Override - public List getURLsFromPage(Document page) { - List imageURLs = new ArrayList<>(); - for (Element thumb : page.select("div.item.asset img")) { - String image = thumb.attr("src"); - image = image - .replaceAll("/medium/", "/large/") - .replaceAll("/small/", "/large/"); - imageURLs.add(image); - } - return imageURLs; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } - -} \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DrawcrowdRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DrawcrowdRipperTest.java deleted file mode 100644 index b326d365..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DrawcrowdRipperTest.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URL; - -import com.rarchives.ripme.ripper.rippers.DrawcrowdRipper; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -public class DrawcrowdRipperTest extends RippersTest { - @Test - @Disabled("https://github.com/RipMeApp/ripme/issues/304 -- Drawcrowd broken (site changed)") - public void testRip() throws IOException { - DrawcrowdRipper ripper = new DrawcrowdRipper(new URL("https://drawcrowd.com/rabbiteyes")); - testRipper(ripper); - } - -} From 82ef20ac74dc318ae7c3e14b2847ff1b5696fbb7 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 11 Apr 2021 19:10:55 +0200 Subject: [PATCH 183/512] remove PorncomixDotOneRipper, #24 --- .../ripper/rippers/PorncomixDotOneRipper.java | 68 ------------------- .../rippers/PorncomixDotOneRipperTest.java | 15 ---- 2 files changed, 83 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixDotOneRipper.java delete mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixDotOneRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixDotOneRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixDotOneRipper.java deleted file mode 100644 index c1e7fac7..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixDotOneRipper.java +++ /dev/null @@ -1,68 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; - -public class PorncomixDotOneRipper extends AbstractHTMLRipper { - - public PorncomixDotOneRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "porncomix"; - } - - @Override - public String getDomain() { - return "porncomix.one"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("https?://www.porncomix.one/gallery/([a-zA-Z0-9_\\-]*)/?$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - throw new MalformedURLException("Expected proncomix URL format: " + - "porncomix.one/gallery/comic - got " + url + " instead"); - } - - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - - @Override - public List getURLsFromPage(Document doc) { - List result = new ArrayList<>(); - // We have 2 loops here to cover all the different album types - for (Element el : doc.select(".dgwt-jg-item > a")) { - result.add(el.attr("href")); - } - for (Element el : doc.select(".unite-gallery > img")) { - result.add(el.attr("data-image")); - - } - return result; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } -} \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixDotOneRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixDotOneRipperTest.java deleted file mode 100644 index 9d4df122..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixDotOneRipperTest.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URL; - -import com.rarchives.ripme.ripper.rippers.PorncomixDotOneRipper; -import org.junit.jupiter.api.Test; - -public class PorncomixDotOneRipperTest extends RippersTest { - @Test - public void testPorncomixAlbum() throws IOException { - PorncomixDotOneRipper ripper = new PorncomixDotOneRipper(new URL("https://www.porncomix.one/gallery/blacknwhite-make-america-great-again")); - testRipper(ripper); - } -} \ No newline at end of file From 3b6727192fce3164c71435b84f1257b9dfb7b7ed Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 6 Apr 2021 20:36:31 +0200 Subject: [PATCH 184/512] redgifrippertest flaky --- .../rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java index 8b45594d..ed71128d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java @@ -36,6 +36,7 @@ public class RedgifsRipperTest extends RippersTest { * @throws IOException */ @Test + @Tag("flaky") public void testRedgifsProfile() throws IOException { RedgifsRipper ripper = new RedgifsRipper(new URL("https://redgifs.com/users/margo_monty")); testRipper(ripper); From 6aa7882dd8a37c4102573c5355c638750371a04f Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 14 Mar 2021 09:11:47 +0100 Subject: [PATCH 185/512] hetaicafe flaky --- .../ripme/tst/ripper/rippers/HentaicafeRipperTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java index 9c922260..555c2662 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java @@ -4,16 +4,19 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.HentaiCafeRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class HentaicafeRipperTest extends RippersTest { @Test + @Tag("flaky") public void testHentaiCafeAlbum() throws IOException { HentaiCafeRipper ripper = new HentaiCafeRipper(new URL("https://hentai.cafe/kikuta-the-oni-in-the-room/")); testRipper(ripper); } // This album has a line break (
) in the url. Test it to make sure ripme can handle these invalid urls @Test + @Tag("flaky") public void testAlbumWithInvalidChars() throws IOException { HentaiCafeRipper ripper = new HentaiCafeRipper(new URL("https://hentai.cafe/chobipero-club/")); testRipper(ripper); From ff7033dcf91c25aa322a887de8b5b62403c597b8 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 14 Mar 2021 09:27:07 +0100 Subject: [PATCH 186/512] smuttyripper flaky --- .../rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java index cb1a78c4..4085bb56 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java @@ -5,10 +5,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.SmuttyRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class SmuttyRipperTest extends RippersTest { @Test + @Tag("flaky") public void testRip() throws IOException { SmuttyRipper ripper = new SmuttyRipper(new URL("https://smutty.com/user/QUIGON/")); testRipper(ripper); From 28cee940b0ce2efcb800b0a0797239950267a753 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 20 Mar 2021 10:01:47 +0100 Subject: [PATCH 187/512] pornhubrippertest flaky --- .../rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java index 354b4e62..22de8349 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java @@ -8,6 +8,7 @@ import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; import org.jsoup.nodes.Document; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class PornhubRipperTest extends RippersTest { @@ -28,6 +29,7 @@ public class PornhubRipperTest extends RippersTest { } @Test + @Tag("flaky") public void testGetNextPage() throws IOException { String baseURL = "https://www.pornhub.com/album/30687901"; PornhubRipper ripper = new PornhubRipper(new URL(baseURL)); From d2e248cb6160252f787b7446fe4f3855c86edb53 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 20 Mar 2021 10:10:09 +0100 Subject: [PATCH 188/512] load history should not produce NPE, ripme#1853 see: * https://github.com/RipMeApp/ripme/issues/1853 * https://stackoverflow.com/questions/20714058/file-exists-and-is-directory-but-listfiles-returns-null --- .../java/com/rarchives/ripme/ui/MainWindow.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 92b8071c..48e8d836 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1183,13 +1183,15 @@ public final class MainWindow implements Runnable, RipStatusHandler { // Guess rip history based on rip folder String[] dirs = Utils.getWorkingDirectory() .list((dir, file) -> new File(dir.getAbsolutePath() + File.separator + file).isDirectory()); - for (String dir : dirs) { - String url = RipUtils.urlFromDirectoryName(dir); - if (url != null) { - // We found one, add it to history - HistoryEntry entry = new HistoryEntry(); - entry.url = url; - HISTORY.add(entry); + if (dirs != null) { + for (String dir : dirs) { + String url = RipUtils.urlFromDirectoryName(dir); + if (url != null) { + // We found one, add it to history + HistoryEntry entry = new HistoryEntry(); + entry.url = url; + HISTORY.add(entry); + } } } } From d11fdb6652f217453f176493bcd4d7097aaf5f62 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Mar 2021 08:35:55 +0200 Subject: [PATCH 189/512] AerisdiesRipperTest flaky --- .../ripme/tst/ripper/rippers/AerisdiesRipperTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java index d3166240..c4c2a7a8 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java @@ -6,16 +6,19 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.AerisdiesRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class AerisdiesRipperTest extends RippersTest { @Test + @Tag("flaky") public void testAlbum() throws IOException { AerisdiesRipper ripper = new AerisdiesRipper(new URL("http://www.aerisdies.com/html/lb/alb_1097_1.html")); testRipper(ripper); } @Test + @Tag("flaky") public void testSubAlbum() throws IOException { AerisdiesRipper ripper = new AerisdiesRipper(new URL("http://www.aerisdies.com/html/lb/alb_3692_1.html")); testRipper(ripper); From 6e6648bf66f22d7b55bf89dd4c6b32f5c0de1480 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 23 Jan 2021 12:39:53 +0100 Subject: [PATCH 190/512] hentai2 test flaky --- .../ripme/tst/ripper/rippers/Hentai2readRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java index f448f0de..c6e2d3de 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.Hentai2readRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class Hentai2readRipperTest extends RippersTest { @Test + @Tag("flaky") public void testHentai2readAlbum() throws IOException { Hentai2readRipper ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/1/")); testRipper(ripper); From c387460a9527a7a1cacee603d633d27ddbf078e6 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 1 May 2021 08:20:27 +0200 Subject: [PATCH 191/512] zizkiripper either broken or flaky --- .../ripme/tst/ripper/rippers/ZizkiRipperTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java index 7af10e55..3d21df97 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java @@ -5,20 +5,27 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.ZizkiRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class ZizkiRipperTest extends RippersTest { + + @Test + @Tag("flaky") public void testRip() throws IOException { ZizkiRipper ripper = new ZizkiRipper(new URL("http://zizki.com/dee-chorde/we-got-spirit")); testRipper(ripper); } + @Test public void testGetGID() throws IOException { URL url = new URL("http://zizki.com/dee-chorde/we-got-spirit"); ZizkiRipper ripper = new ZizkiRipper(url); Assertions.assertEquals("dee-chorde", ripper.getGID(url)); } + @Test + @Tag("flaky") public void testAlbumTitle() throws IOException { URL url = new URL("http://zizki.com/dee-chorde/we-got-spirit"); ZizkiRipper ripper = new ZizkiRipper(url); From dfb217179bc6d7f410d6fcefd9a53af4dc4ecae9 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 23 Jan 2021 20:19:28 +0100 Subject: [PATCH 192/512] imagebam, motherless tests flaky --- .../rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java | 2 ++ .../ripme/tst/ripper/rippers/MotherlessRipperTest.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java index efe57b96..5ecfe3f6 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ImagebamRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class ImagebamRipperTest extends RippersTest { @Test + @Tag("flaky") public void testImagebamRip() throws IOException { ImagebamRipper ripper = new ImagebamRipper(new URL("http://www.imagebam.com/gallery/488cc796sllyf7o5srds8kpaz1t4m78i")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java index 2739f9da..97f48a5f 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java @@ -5,10 +5,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.MotherlessRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class MotherlessRipperTest extends RippersTest { @Test + @Tag("flaky") public void testMotherlessAlbumRip() throws IOException { MotherlessRipper ripper = new MotherlessRipper(new URL("https://motherless.com/G1168D90")); testRipper(ripper); From 0b63c26c8247d09f046bb8ae07453850adf31846 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Feb 2021 11:26:00 +0100 Subject: [PATCH 193/512] hentainexus flaky --- .../ripme/tst/ripper/rippers/HentainexusRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java index a244276c..00340eba 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java @@ -8,10 +8,12 @@ import java.util.List; import com.rarchives.ripme.ripper.rippers.HentaiNexusRipper; import org.json.JSONObject; import org.junit.Assert; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class HentainexusRipperTest extends RippersTest { @Test + @Tag("flaky") public void testHentaiNexusJson() throws IOException { List testURLs = new ArrayList<>(); testURLs.add(new URL("https://hentainexus.com/view/9202")); From 2d235fd490fb892a540182655d3600ea0aa4a3e2 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Feb 2021 15:05:25 +0100 Subject: [PATCH 194/512] ArtStationRipperTest flaky --- .../ripme/tst/ripper/rippers/ArtStationRipperTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java index d7cf6cdf..e29a32ed 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java @@ -7,12 +7,13 @@ import java.util.List; import com.rarchives.ripme.ripper.rippers.ArtStationRipper; -import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class ArtStationRipperTest extends RippersTest { @Test + @Tag("flaky") public void testArtStationProjects() throws IOException { List contentURLs = new ArrayList<>(); contentURLs.add(new URL("https://www.artstation.com/artwork/the-dwarf-mortar")); @@ -25,7 +26,7 @@ public class ArtStationRipperTest extends RippersTest { } @Test - @Disabled("Failed with cloudflare protection") + @Tag("flaky") public void testArtStationUserProfiles() throws IOException { List contentURLs = new ArrayList<>(); contentURLs.add(new URL("https://www.artstation.com/heitoramatsu")); From 1911fb1d13752812e6ddb96a79feba05e47c6585 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 9 May 2021 19:11:18 +0200 Subject: [PATCH 195/512] nhentairippertest flaky --- .../rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java index b7e1a968..a1872703 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java @@ -7,6 +7,7 @@ import java.util.List; import com.rarchives.ripme.ripper.rippers.NhentaiRipper; import com.rarchives.ripme.utils.RipUtils; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class NhentaiRipperTest extends RippersTest { @@ -22,6 +23,7 @@ public class NhentaiRipperTest extends RippersTest { // Test the tag black listing @Test + @Tag("flaky") public void testTagBlackList() throws IOException { URL url = new URL("https://nhentai.net/g/233295/"); NhentaiRipper ripper = new NhentaiRipper(url); From 3e47be752e79cab015faede90a14b54172cfbe6b Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 16 Jan 2021 18:18:15 +0100 Subject: [PATCH 196/512] java-15 in test, jacoco-0.86 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 37136d8e..15b06823 100644 --- a/pom.xml +++ b/pom.xml @@ -132,7 +132,7 @@ org.jacoco jacoco-maven-plugin - 0.8.5 + 0.8.6 prepare-agent From 0d568069a95049f6a2365b46cd1b5ddc1d610b9a Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 26 Nov 2020 22:37:06 +0100 Subject: [PATCH 197/512] one build is sufficient to test newest java check builds are flaky, and doing them more often in a short time frame makes them even more flaky. --- .github/workflows/maven.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index bb44b0c8..32a50797 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -9,7 +9,10 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - java: [1.8, 1.14] + java: [1.8] + include: # test newest java on one os only + - os: ubuntu-latest + java: 1.14 steps: - uses: actions/checkout@v1 From 08b696685e871139575ea6b4446de831e2295015 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 30 Jan 2021 11:05:15 +0100 Subject: [PATCH 198/512] test with java-15 --- .github/workflows/maven.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 32a50797..b48244ed 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -12,7 +12,8 @@ jobs: java: [1.8] include: # test newest java on one os only - os: ubuntu-latest - java: 1.14 + java: 1.15 + steps: - uses: actions/checkout@v1 From e90e6af9f758d86e8581aa938c57f9ef03ab2a33 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 9 May 2021 19:11:18 +0200 Subject: [PATCH 199/512] nhentairippertest flaky --- .../rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java index b7e1a968..a1872703 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java @@ -7,6 +7,7 @@ import java.util.List; import com.rarchives.ripme.ripper.rippers.NhentaiRipper; import com.rarchives.ripme.utils.RipUtils; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class NhentaiRipperTest extends RippersTest { @@ -22,6 +23,7 @@ public class NhentaiRipperTest extends RippersTest { // Test the tag black listing @Test + @Tag("flaky") public void testTagBlackList() throws IOException { URL url = new URL("https://nhentai.net/g/233295/"); NhentaiRipper ripper = new NhentaiRipper(url); From d0e2861a15ef78275ca07d3b9ec5f6677ef9e1e8 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 14 May 2021 06:16:00 +0200 Subject: [PATCH 200/512] gradle-7.0.1 --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index d8442f55..625381b9 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.0-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.0.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists \ No newline at end of file From 21ad0cc4451a3f628c77dcd00a42d203ab7fa2e5 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 14 May 2021 12:05:44 +0200 Subject: [PATCH 201/512] 2 tests flaky --- .../ripme/tst/ripper/rippers/AllporncomicRipperTest.java | 2 ++ .../ripme/tst/ripper/rippers/DynastyscansRipperTest.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AllporncomicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AllporncomicRipperTest.java index f8466072..41c9542a 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AllporncomicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AllporncomicRipperTest.java @@ -1,6 +1,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.AllporncomicRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -8,6 +9,7 @@ import java.net.URL; public class AllporncomicRipperTest extends RippersTest { @Test + @Tag("flaky") public void testAlbum() throws IOException { AllporncomicRipper ripper = new AllporncomicRipper(new URL("https://allporncomic.com/porncomic/dnd-pvp-dungeons-dragons-fred-perry/1-dnd-pvp")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java index 4c8d6416..fb920545 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java @@ -5,10 +5,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.DynastyscansRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class DynastyscansRipperTest extends RippersTest { @Test + @Tag("flaky") public void testRip() throws IOException { DynastyscansRipper ripper = new DynastyscansRipper(new URL("https://dynasty-scans.com/chapters/under_one_roof_ch01")); testRipper(ripper); From ee18e2a1a7a63693fcb2b1c1f6a85bfc4895d39b Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 14 May 2021 21:08:20 +0200 Subject: [PATCH 202/512] readme, add compose-jb short explaination --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index ed630525..952b91fa 100644 --- a/README.md +++ b/README.md @@ -116,3 +116,12 @@ Please note that some tests may fail as sites change and our rippers become out of date. Start by building and testing a released version of RipMe and then ensure that any changes you make do not cause more tests to break. + +# New GUI - compose-jb +As Java Swing will go away in future, a new GUI technology should be used. One of the +candidates is [Jetpack Compose for Desktop](https://github.com/JetBrains/compose-jb/). + +The library leverages the compose library for android and provides it for android, +desktop and web. The navigation library is not available for desktop, so Arkadii Ivanov +implemented +[decompose](https://proandroiddev.com/a-comprehensive-hundred-line-navigation-for-jetpack-desktop-compose-5b723c4f256e). From c14d068f09dfafeed678cddb30618432261c19f3 Mon Sep 17 00:00:00 2001 From: Stilgar Date: Thu, 20 May 2021 16:16:32 +0200 Subject: [PATCH 203/512] Fix #1868 Imagebam Error : Change Layout --- .../ripme/ripper/rippers/ImagebamRipper.java | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index 3aca67cf..90ad05a0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -11,6 +11,8 @@ import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; + +import org.apache.commons.lang.StringUtils; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -80,7 +82,7 @@ public class ImagebamRipper extends AbstractHTMLRipper { @Override public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList<>(); - for (Element thumb : doc.select("div > a[target=_blank]:not(.footera)")) { + for (Element thumb : doc.select("div > a[class=thumbnail]:not(.footera)")) { imageURLs.add(thumb.attr("href")); } return imageURLs; @@ -97,15 +99,12 @@ public class ImagebamRipper extends AbstractHTMLRipper { public String getAlbumTitle(URL url) throws MalformedURLException { try { // Attempt to use album title as GID - Elements elems = getFirstPage().select("legend"); + Elements elems = getFirstPage().select("[id=gallery-name]"); String title = elems.first().text(); LOGGER.info("Title text: '" + title + "'"); - Pattern p = Pattern.compile("^(.*)\\s\\d* image.*$"); - Matcher m = p.matcher(title); - if (m.matches()) { - return getHost() + "_" + getGID(url) + " (" + m.group(1).trim() + ")"; + if (StringUtils.isNotBlank(title)) { + return getHost() + "_" + getGID(url) + " (" + title + ")"; } - LOGGER.info("Doesn't match " + p.pattern()); } catch (Exception e) { // Fall back to default album naming convention LOGGER.warn("Failed to get album title from " + url, e); @@ -143,14 +142,9 @@ public class ImagebamRipper extends AbstractHTMLRipper { Elements metaTags = doc.getElementsByTag("meta"); String imgsrc = "";//initialize, so no NullPointerExceptions should ever happen. - - for (Element metaTag: metaTags) { - //the direct link to the image seems to always be linked in the part of the html. - if (metaTag.attr("property").equals("og:image")) { - imgsrc = metaTag.attr("content"); - LOGGER.info("Found URL " + imgsrc); - break;//only one (useful) image possible for an "image page". - } + Elements elem = doc.select("img[class*=main-image]"); + if ((elem != null) && (elem.size() > 0)) { + imgsrc = elem.first().attr("src"); } //for debug, or something goes wrong. From 88bf1a01375058dc3d0bb51a981021b446086a84 Mon Sep 17 00:00:00 2001 From: Stilgar Date: Fri, 21 May 2021 00:31:00 +0200 Subject: [PATCH 204/512] Fix YouPorn Test (update test URL) Video cannot be loaded: "Video has been flagged for verification" --- .../rarchives/ripme/tst/ripper/rippers/YoupornRipperTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YoupornRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YoupornRipperTest.java index bce22d62..68bf2b69 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YoupornRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YoupornRipperTest.java @@ -14,7 +14,9 @@ public class YoupornRipperTest extends RippersTest { @Tag("flaky") public void testYoupornRipper() throws IOException { List contentURLs = new ArrayList<>(); - contentURLs.add(new URL("http://www.youporn.com/watch/7669155/mrs-li-amateur-69-orgasm/?from=categ")); + // Video cannot be loaded: "Video has been flagged for verification" + //contentURLs.add(new URL("http://www.youporn.com/watch/7669155/mrs-li-amateur-69-orgasm/?from=categ")); + contentURLs.add(new URL("https://www.youporn.com/watch/13158849/smashing-star-slut-part-2/")); for (URL url : contentURLs) { YoupornRipper ripper = new YoupornRipper(url); testRipper(ripper); From df5a5786cdecf43c44ea5d1ffecbefce83f5a7f7 Mon Sep 17 00:00:00 2001 From: Stilgar Date: Fri, 21 May 2021 00:35:54 +0200 Subject: [PATCH 205/512] Fix Hentai nexus and Hentai cafe (disable test), both offline --- .../ripme/tst/ripper/rippers/HentaicafeRipperTest.java | 3 +++ .../ripme/tst/ripper/rippers/HentainexusRipperTest.java | 2 ++ 2 files changed, 5 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java index 555c2662..03a96991 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java @@ -4,12 +4,14 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.HentaiCafeRipper; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class HentaicafeRipperTest extends RippersTest { @Test @Tag("flaky") + @Disabled("20/05/2021 This test was disabled as the site has experienced notable downtime") public void testHentaiCafeAlbum() throws IOException { HentaiCafeRipper ripper = new HentaiCafeRipper(new URL("https://hentai.cafe/kikuta-the-oni-in-the-room/")); testRipper(ripper); @@ -17,6 +19,7 @@ public class HentaicafeRipperTest extends RippersTest { // This album has a line break (
) in the url. Test it to make sure ripme can handle these invalid urls @Test @Tag("flaky") + @Disabled("20/05/2021 This test was disabled as the site has experienced notable downtime") public void testAlbumWithInvalidChars() throws IOException { HentaiCafeRipper ripper = new HentaiCafeRipper(new URL("https://hentai.cafe/chobipero-club/")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java index 00340eba..cad4bb7e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java @@ -8,12 +8,14 @@ import java.util.List; import com.rarchives.ripme.ripper.rippers.HentaiNexusRipper; import org.json.JSONObject; import org.junit.Assert; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class HentainexusRipperTest extends RippersTest { @Test @Tag("flaky") + @Disabled("20/05/2021 This test was disabled as the site has experienced notable downtime") public void testHentaiNexusJson() throws IOException { List testURLs = new ArrayList<>(); testURLs.add(new URL("https://hentainexus.com/view/9202")); From 05f97cfadce1cf3174e3f3a07f7ef702647503d7 Mon Sep 17 00:00:00 2001 From: Stilgar Date: Fri, 21 May 2021 00:46:46 +0200 Subject: [PATCH 206/512] Fix Flaky Tests stop() is called too soon in some cases, download might not have been started and calling stop will leave the items in pending so only call stop if items have been completed or on error + improve shouldStop as AtomicBoolean + trace logging in case issue happens again --- .../rarchives/ripme/ripper/AbstractHTMLRipper.java | 10 +++++----- .../rarchives/ripme/ripper/AbstractJSONRipper.java | 6 +++--- .../com/rarchives/ripme/ripper/AbstractRipper.java | 11 +++++++---- .../java/com/rarchives/ripme/ripper/AlbumRipper.java | 6 +++--- .../ripme/tst/ripper/rippers/RippersTest.java | 7 +++++++ 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 3e3fdb18..4431f2cf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -128,7 +128,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { index += 1; LOGGER.debug("Found image url #" + index + ": " + imageURL); downloadURL(new URL(imageURL), index); - if (isStopped()) { + if (isStopped() || isThisATest()) { break; } } @@ -139,7 +139,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { if (!textURLs.isEmpty()) { LOGGER.debug("Found description link(s) from " + doc.location()); for (String textURL : textURLs) { - if (isStopped()) { + if (isStopped() || isThisATest()) { break; } textindex += 1; @@ -293,10 +293,10 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { * Queues multiple URLs of single images to download from a single Album URL */ public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. - if (super.isThisATest() && - (itemsPending.size() > 0 || itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { + // Only download one file if this is a test. + if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { stop(); + itemsPending.clear(); return false; } if (!allowDuplicates() diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index d7e93fcb..44596702 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -141,10 +141,10 @@ public abstract class AbstractJSONRipper extends AbstractRipper { * Queues multiple URLs of single images to download from a single Album URL */ public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. - if (super.isThisATest() && - (itemsPending.size() > 0 || itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { + // Only download one file if this is a test. + if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { stop(); + itemsPending.clear(); return false; } if (!allowDuplicates() diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 3653b9f0..465debe5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -14,6 +14,8 @@ import java.util.List; import java.util.Map; import java.util.Observable; import java.util.Scanner; +import java.util.concurrent.atomic.AtomicBoolean; + import org.apache.log4j.FileAppender; import org.apache.log4j.Logger; import org.jsoup.HttpStatusException; @@ -47,17 +49,18 @@ public abstract class AbstractRipper public boolean hasASAPRipping() { return false; } // Everytime addUrlToDownload skips a already downloaded url this increases by 1 public int alreadyDownloadedUrls = 0; - private boolean shouldStop = false; + private final AtomicBoolean shouldStop = new AtomicBoolean(false); private static boolean thisIsATest = false; public void stop() { - shouldStop = true; + LOGGER.trace("stop()"); + shouldStop.set(true); } public boolean isStopped() { - return shouldStop; + return shouldStop.get(); } protected void stopCheck() throws IOException { - if (shouldStop) { + if (shouldStop.get()) { throw new IOException("Ripping interrupted"); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index f433e77f..f245ba62 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -51,10 +51,10 @@ public abstract class AlbumRipper extends AbstractRipper { * Queues multiple URLs of single images to download from a single Album URL */ public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. - if (super.isThisATest() && - (itemsPending.size() > 0 || itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { + // Only download one file if this is a test. + if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { stop(); + itemsPending.clear(); return false; } if (!allowDuplicates() diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java index c09b8018..9e77a605 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java @@ -32,6 +32,13 @@ public class RippersTest { ripper.setup(); ripper.markAsTest(); ripper.rip(); + if (logger.isTraceEnabled()) { + logger.trace("working dir: " + ripper.getWorkingDir()); + logger.trace("list files: " + ripper.getWorkingDir().listFiles().length); + for (int i = 0; i < ripper.getWorkingDir().listFiles().length; i++) { + logger.trace(" " + ripper.getWorkingDir().listFiles()[i]); + } + } Assertions.assertTrue(ripper.getWorkingDir().listFiles().length >= 1, "Failed to download a single file from " + ripper.getURL()); } catch (IOException e) { From d0d1a3dbbb6606064a468fe26d029d1f95d082c4 Mon Sep 17 00:00:00 2001 From: Stilgar Date: Fri, 21 May 2021 00:52:36 +0200 Subject: [PATCH 207/512] Fix Webtoons (disable Age Gate) --- .../com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java index ded3ce2c..d82f4aff 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java @@ -84,7 +84,10 @@ public class WebtoonsRipper extends AbstractHTMLRipper { public Document getFirstPage() throws IOException { Response resp = Http.url(url).response(); cookies = resp.cookies(); - return Http.url(url).get(); + cookies.put("needCOPPA", "false"); + cookies.put("needCCPA", "false"); + cookies.put("needGDPR", "false"); + return Http.url(url).cookies(cookies).get(); } @Override From bffc8863b42d2e67fafeb2d5874e36ff29b45dc3 Mon Sep 17 00:00:00 2001 From: Stilgar Date: Fri, 21 May 2021 00:59:56 +0200 Subject: [PATCH 208/512] Fix WordpressComic Page layout changes to: freeadultcomix.com shipinbottle.pepsaga.com --- .../rarchives/ripme/ripper/rippers/WordpressComicRipper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index 143c396a..0589f29d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -376,7 +376,7 @@ public class WordpressComicRipper extends AbstractHTMLRipper { // freeadultcomix gets it own if because it needs to add http://freeadultcomix.com to the start of each link // TODO review the above comment which no longer applies -- see if there's a refactoring we should do here. if (url.toExternalForm().contains("freeadultcomix.com")) { - for (Element elem : doc.select("div.single-post > p > img.aligncenter")) { + for (Element elem : doc.select("div.post-texto > p > noscript > img[class*=aligncenter]")) { result.add(elem.attr("src")); } } else if (url.toExternalForm().contains("comics-xxx.com")) { @@ -384,7 +384,7 @@ public class WordpressComicRipper extends AbstractHTMLRipper { result.add(elem.attr("src")); } } else if (url.toExternalForm().contains("shipinbottle.pepsaga.com")) { - for (Element elem : doc.select("div#comic > div.comicpane > a > img")) { + for (Element elem : doc.select("div#comic > a > img")) { result.add(elem.attr("src")); } } else if (url.toExternalForm().contains("8muses.download")) { From 735f13cc9f1da5887a24a6b8795e0ca1d6dd1dd6 Mon Sep 17 00:00:00 2001 From: Stilgar Date: Fri, 21 May 2021 01:03:44 +0200 Subject: [PATCH 209/512] Fix Zizki (Page layout change) --- .../com/rarchives/ripme/ripper/rippers/ZizkiRipper.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java index 35733325..fb048a6d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java @@ -87,14 +87,12 @@ public class ZizkiRipper extends AbstractHTMLRipper { if (thumb.hasAttr("typeof")) { img_type = thumb.attr("typeof"); if (img_type.equals("foaf:Image")) { - LOGGER.debug("Found image with " + img_type); if (thumb.parent() != null && - thumb.parent().parent() != null && - thumb.parent().parent().attr("class") != null && - thumb.parent().parent().attr("class").equals("aimage-center") + thumb.parent().attr("class") != null && + thumb.parent().attr("class").contains("colorbox") ) { - src = thumb.attr("src"); + src = thumb.parent().attr("href"); LOGGER.debug("Found url with " + src); if (!src.contains("zizki.com")) { } else { From 36ea142a1985c202afbdfd297f387c8a54a81d19 Mon Sep 17 00:00:00 2001 From: Stilgar Date: Fri, 21 May 2021 01:08:23 +0200 Subject: [PATCH 210/512] Fix slow tests Added fast stop to Furaffinity and Xhamster (break in for loops) --- .../rarchives/ripme/ripper/rippers/FuraffinityRipper.java | 3 +++ .../com/rarchives/ripme/ripper/rippers/XhamsterRipper.java | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java index 683c791b..99a066a6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java @@ -125,6 +125,9 @@ public class FuraffinityRipper extends AbstractHTMLRipper { urls.add(urlToAdd); } } + if (isStopped() || isThisATest()) { + break; + } } return urls; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 7ade1e55..c66400a3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -84,6 +84,9 @@ public class XhamsterRipper extends AbstractHTMLRipper { LOGGER.info("getting albums"); for (Element elem : doc.select("div.item-container > a.item")) { urlsToAddToQueue.add(elem.attr("href")); + if (isStopped() || isThisATest()) { + break; + } } LOGGER.info(doc.html()); return urlsToAddToQueue; @@ -169,6 +172,9 @@ public class XhamsterRipper extends AbstractHTMLRipper { } catch (IOException e) { LOGGER.error("Was unable to load page " + pageWithImageUrl); } + if (isStopped() || isThisATest()) { + break; + } } } else { String imgUrl = doc.select("div.player-container > a").attr("href"); From 4a1e1256a8a4caf5c217a27ed760730e02574d79 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 22 May 2021 07:54:38 +0200 Subject: [PATCH 211/512] gradle-7.0.2 --- gradle/wrapper/gradle-wrapper.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 625381b9..0f80bbf5 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.0.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.0.2-bin.zip zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists \ No newline at end of file +zipStorePath=wrapper/dists From 8b3b60c105e5e9eb2d33dc438adaa53b920563bb Mon Sep 17 00:00:00 2001 From: schar Date: Mon, 24 May 2021 22:39:19 +0530 Subject: [PATCH 212/512] Added extra condition to include reddit gallery gifs urls --- .../com/rarchives/ripme/ripper/rippers/RedditRipper.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 65d854fb..09569fc7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -318,7 +318,12 @@ public class RedditRipper extends AlbumRipper { prefix += String.format("%02d-", i + 1); } try { - URL mediaURL = new URL(media.getJSONObject("s").getString("u").replaceAll("&", "&")); + URL mediaURL; + if (!media.getJSONObject("s").isNull("gif")) { + mediaURL = new URL(media.getJSONObject("s").getString("gif").replaceAll("&", "&")); + } else { + mediaURL = new URL(media.getJSONObject("s").getString("u").replaceAll("&", "&")); + } addURLToDownload(mediaURL, prefix, subdirectory); } catch (MalformedURLException | JSONException e) { LOGGER.error("[!] Unable to parse gallery JSON:\ngallery_data:\n" + data +"\nmedia_metadata:\n" + metadata); From 9494f9ba43818d80629c883caa0a3ae9d1f97c0b Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 4 Jun 2021 13:37:22 +0200 Subject: [PATCH 213/512] update dependency versions --- build.gradle.kts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 6edf7f77..f8ef8b63 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -16,16 +16,16 @@ dependencies { implementation("org.jsoup:jsoup:1.8.1") implementation("org.json:json:20190722") implementation("commons-configuration:commons-configuration:1.7") - implementation("commons-cli:commons-cli:1.2") - implementation("commons-io:commons-io:2.7") - implementation("org.apache.httpcomponents:httpclient:4.3.6") - implementation("org.apache.httpcomponents:httpmime:4.3.3") + implementation("commons-cli:commons-cli:1.4") + implementation("commons-io:commons-io:2.9.0") + implementation("org.apache.httpcomponents:httpclient:4.5.13") + implementation("org.apache.httpcomponents:httpmime:4.5.13") implementation("org.apache.logging.log4j:log4j-api:2.14.1") implementation("org.apache.logging.log4j:log4j-core:2.14.1") - implementation("org.graalvm.js:js:20.1.0") - testImplementation(enforcedPlatform("org.junit:junit-bom:5.6.2")) + implementation("org.graalvm.js:js:21.1.0") + testImplementation(enforcedPlatform("org.junit:junit-bom:5.7.2")) testImplementation("org.junit.jupiter:junit-jupiter") - testImplementation("junit:junit:4.13") + testImplementation("junit:junit:4.13.2") } group = "com.rarchives.ripme" From 9a05be80a573cc356985135cdf38ace5fc5aca9a Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 6 Jun 2021 16:14:23 +0200 Subject: [PATCH 214/512] meituri is tujigu RipMeApp#1894 https://github.com/RipMeApp/ripme/issues/1894 --- .../ripme/ripper/rippers/MeituriRipper.java | 16 ++++++++-------- .../tst/ripper/rippers/MeituriRipperTest.java | 5 ++--- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MeituriRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MeituriRipper.java index 8bdd2b2f..d5c198bb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MeituriRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MeituriRipper.java @@ -21,12 +21,12 @@ public class MeituriRipper extends AbstractHTMLRipper { @Override public String getHost() { - return "meituri"; + return "tujigu"; } @Override public String getDomain() { - return "meituri.com"; + return "tujigu.com"; } // To use in getting URLs @@ -35,18 +35,18 @@ public class MeituriRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { // without escape - // ^https?://[w.]*meituri\.com/a/([0-9]+)/([0-9]+\.html)*$ - // https://www.meituri.com/a/14449/ - // also matches https://www.meituri.com/a/14449/3.html etc. + // ^https?://[w.]*tujigu\.com/a/([0-9]+)/([0-9]+\.html)*$ + // https://www.tujigu.com/a/14449/ + // also matches https://www.tujigu.com/a/14449/3.html etc. // group 1 is 14449 - Pattern p = Pattern.compile("^https?://[w.]*meituri\\.com/a/([0-9]+)/([0-9]+\\.html)*$"); + Pattern p = Pattern.compile("^https?://[w.]*tujigu\\.com/a/([0-9]+)/([0-9]+\\.html)*$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { albumID = m.group(1); return m.group(1); } throw new MalformedURLException( - "Expected meituri.com URL format: " + "meituri.com/a/albumid/ - got " + url + "instead"); + "Expected tujigu.com URL format: " + "tujigu.com/a/albumid/ - got " + url + "instead"); } @Override @@ -71,7 +71,7 @@ public class MeituriRipper extends AbstractHTMLRipper { } // Base URL: http://ii.hywly.com/a/1/albumid/imgnum.jpg - String baseURL = "http://ii.hywly.com/a/1/" + albumID + "/"; + String baseURL = "https://tjg.hywly.com/a/1/" + albumID + "/"; // Loop through and add images to the URL list for (int i = 1; i <= numOfImages; i++) { diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MeituriRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MeituriRipperTest.java index 883b73e3..d34325e3 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MeituriRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MeituriRipperTest.java @@ -11,15 +11,14 @@ import org.junit.jupiter.api.Test; public class MeituriRipperTest extends RippersTest { @Test - @Disabled("Broken ripper") public void testMeituriRip() throws IOException { - MeituriRipper ripper = new MeituriRipper(new URL("https://www.meituri.com/a/14449/")); + MeituriRipper ripper = new MeituriRipper(new URL("https://www.tujigu.com/a/14449/")); testRipper(ripper); } @Test public void testGetGID() throws IOException { - URL url = new URL("https://www.meituri.com/a/14449/"); + URL url = new URL("https://www.tujigu.com/a/14449/"); MeituriRipper ripper = new MeituriRipper(url); Assertions.assertEquals("14449", ripper.getGID(url)); } From 4e7092f1a779e6559fd704d9dc6538d4c8d45b3f Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Mar 2021 09:04:04 +0200 Subject: [PATCH 215/512] xhamster next page link a.prev-next-list-link--next multiple issues reported, like #20, #21, RipMeApp#1852. --- .../java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 226dec3c..c5ffe537 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -142,7 +142,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException { if (doc.select("a.prev-next-list-link").first() != null) { - String nextPageUrl = doc.select("a.prev-next-list-link").first().attr("href"); + String nextPageUrl = doc.select("a.prev-next-list-link--next").first().attr("href"); if (nextPageUrl.startsWith("http")) { nextPageUrl = nextPageUrl.replaceAll("https?://\\w?\\w?\\.?xhamster([^<]*)\\.", "https://m.xhamster$1."); return Http.url(nextPageUrl).get(); From 958d0933ba729c1eed792e0200c71b30fc27366d Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 28 Mar 2021 09:07:54 +0200 Subject: [PATCH 216/512] xhamster css selector shortened issues like #20. #21 may be solved. --- .../java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index c5ffe537..cf632dcb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -159,7 +159,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { if (!isVideoUrl(url)) { if (!doc.select("div.picture_view > div.pictures_block > div.items > div.item-container > a.item").isEmpty()) { // Old HTML structure is still present at some places - for (Element page : doc.select("div.picture_view > div.pictures_block > div.items > div.item-container > a.item")) { + for (Element page : doc.select(".clearfix > div > a.slided")) { // Make sure we don't waste time running the loop if the ripper has been stopped if (isStopped()) { break; From 835207b73c4f40493c773e5664c4b430f5464bb9 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 25 Jun 2021 01:24:50 +0200 Subject: [PATCH 217/512] imagearn does not exist any more --- .../ripme/ripper/rippers/ImagearnRipper.java | 112 ------------------ .../ripper/rippers/ImagearnRipperTest.java | 15 --- 2 files changed, 127 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java delete mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java deleted file mode 100644 index 062217b2..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagearnRipper.java +++ /dev/null @@ -1,112 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; - -public class ImagearnRipper extends AbstractHTMLRipper { - - public ImagearnRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "imagearn"; - } - @Override - public String getDomain() { - return "imagearn.com"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^.*imagearn.com/+gallery.php\\?id=([0-9]+).*$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - throw new MalformedURLException( - "Expected imagearn.com gallery formats: " - + "imagearn.com/gallery.php?id=####..." - + " Got: " + url); - } - - public URL sanitizeURL(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^.*imagearn.com/+image.php\\?id=[0-9]+.*$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - // URL points to imagearn *image*, not gallery - try { - url = getGalleryFromImage(url); - } catch (Exception e) { - LOGGER.error("[!] " + e.getMessage(), e); - } - } - return url; - } - - private URL getGalleryFromImage(URL url) throws IOException { - Document doc = Http.url(url).get(); - for (Element link : doc.select("a[href~=^gallery\\.php.*$]")) { - LOGGER.info("LINK: " + link.toString()); - if (link.hasAttr("href") - && link.attr("href").contains("gallery.php")) { - url = new URL("http://imagearn.com/" + link.attr("href")); - LOGGER.info("[!] Found gallery from given link: " + url); - return url; - } - } - throw new IOException("Failed to find gallery at URL " + url); - } - - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - - @Override - public String getAlbumTitle(URL url) throws MalformedURLException { - try { - Document doc = getFirstPage(); - String title = doc.select("h3 > strong").first().text(); // profile name - return getHost() + "_" + title + "_" + getGID(url); - } catch (Exception e) { - // Fall back to default album naming convention - LOGGER.warn("Failed to get album title from " + url, e); - } - return super.getAlbumTitle(url); - } - - @Override - public List getURLsFromPage(Document doc) { - List imageURLs = new ArrayList<>(); - for (Element thumb : doc.select("div#gallery > div > a")) { - String imageURL = thumb.attr("href"); - try { - Document imagedoc = new Http("http://imagearn.com/" + imageURL).get(); - String image = imagedoc.select("a.thickbox").first().attr("href"); - imageURLs.add(image); - } catch (IOException e) { - LOGGER.warn("Was unable to download page: " + imageURL); - } - } - return imageURLs; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - sleep(1000); - } -} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java deleted file mode 100644 index 69b6d899..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagearnRipperTest.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URL; - -import com.rarchives.ripme.ripper.rippers.ImagearnRipper; -import org.junit.jupiter.api.Test; - -public class ImagearnRipperTest extends RippersTest { - @Test - public void testImagearnRip() throws IOException { - ImagearnRipper ripper = new ImagearnRipper(new URL("http://imagearn.com//gallery.php?id=578682")); - testRipper(ripper); - } -} From 845ea2721ba59a33b2141a5a899c24e5c9e9e7f8 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 25 Jun 2021 01:28:19 +0200 Subject: [PATCH 218/512] flaky tests --- .../rarchives/ripme/tst/ripper/rippers/MeituriRipperTest.java | 2 ++ .../ripme/tst/ripper/rippers/SpankBangRipperTest.java | 2 ++ .../ripme/tst/ripper/rippers/XhamsterRipperTest.java | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MeituriRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MeituriRipperTest.java index d34325e3..eb0970fd 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MeituriRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MeituriRipperTest.java @@ -7,10 +7,12 @@ import com.rarchives.ripme.ripper.rippers.MeituriRipper; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class MeituriRipperTest extends RippersTest { @Test + @Tag("flaky") public void testMeituriRip() throws IOException { MeituriRipper ripper = new MeituriRipper(new URL("https://www.tujigu.com/a/14449/")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SpankBangRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SpankBangRipperTest.java index c73a244e..c655bcf3 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SpankBangRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SpankBangRipperTest.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.SpankbangRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class SpankBangRipperTest extends RippersTest { @Test + @Tag("flaky") public void testSpankBangVideo() throws IOException { SpankbangRipper ripper = new SpankbangRipper(new URL("https://spankbang.com/2a7fh/video/mdb901")); //most popular video of all time on site; should stay up testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java index aaccf47c..7c5aed9f 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java @@ -41,14 +41,14 @@ public class XhamsterRipperTest extends RippersTest { testRipper(ripper); } @Test - @Disabled("ripper broken?") + @Tag("flaky") public void testXhamsterVideo() throws IOException { XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/videos/brazzers-busty-big-booty-milf-lisa-ann-fucks-her-masseur-1492828")); testRipper(ripper); } @Test public void testBrazilianXhamster() throws IOException { - XhamsterRipper ripper = new XhamsterRipper(new URL("https://pt.xhamster.com/photos/gallery/silvana-7105696")); + XhamsterRipper ripper = new XhamsterRipper(new URL("https://pt.xhamster.com/photos/gallery/cartoon-babe-15786301")); testRipper(ripper); } @Test From 0e5267ad7a7fb47bf1c7ba12b095a1242d67be64 Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 29 Jun 2021 06:46:41 +0200 Subject: [PATCH 219/512] imagebam gallery regex: (gallery|view) fix https://github.com/RipMeApp/ripme/issues/1900 --- .../java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index 90ad05a0..e233c03d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -47,7 +47,7 @@ public class ImagebamRipper extends AbstractHTMLRipper { Pattern p; Matcher m; - p = Pattern.compile("^https?://[wm.]*imagebam.com/gallery/([a-zA-Z0-9]+).*$"); + p = Pattern.compile("^https?://[wm.]*imagebam.com/(gallery|view)/([a-zA-Z0-9]+).*$"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); From 28b40d0a4c767afade68e449849303e0e35a01c6 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 11 Sep 2021 07:05:19 +0200 Subject: [PATCH 220/512] update to gradle-7.2 --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 0f80bbf5..ffed3a25 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.0.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 30f4cf3a3956e6f852395614589885db5b892f7f Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 11 Sep 2021 07:31:15 +0200 Subject: [PATCH 221/512] switch off Aerisdies, ArtAlley, Thechive tests --- .../ripme/tst/ripper/rippers/AerisdiesRipperTest.java | 2 ++ .../rarchives/ripme/tst/ripper/rippers/ArtAlleyRipperTest.java | 2 ++ .../rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java | 3 +++ 3 files changed, 7 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java index c4c2a7a8..a8fec2e7 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java @@ -25,12 +25,14 @@ public class AerisdiesRipperTest extends RippersTest { } @Test + @Tag("flaky") public void testDjAlbum() throws IOException { AerisdiesRipper ripper = new AerisdiesRipper(new URL("http://www.aerisdies.com/html/lb/douj_5230_1.html")); testRipper(ripper); } @Test + @Tag("flaky") public void testGetGID() throws IOException { URL url = new URL("http://www.aerisdies.com/html/lb/douj_5230_1.html"); AerisdiesRipper ripper = new AerisdiesRipper(url); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtAlleyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtAlleyRipperTest.java index 693ce619..400e95c3 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtAlleyRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtAlleyRipperTest.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ArtAlleyRipper; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class ArtAlleyRipperTest extends RippersTest { @Test + @Disabled("website switched off") public void testRip() throws IOException { ArtAlleyRipper ripper = new ArtAlleyRipper(new URL("https://artalley.social/@curator/media")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java index 1067f1eb..a0093213 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java @@ -26,6 +26,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.ThechiveRipper; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -43,6 +44,7 @@ public class ThechiveRipperTest extends RippersTest { * @throws IOException */ @Test + @Tag("flaky") public void testTheChiveRip() throws IOException { ThechiveRipper ripper = new ThechiveRipper(new URL( "https://thechive.com/2019/03/16/beautiful-badasses-lookin-good-in-and-out-of-uniform-35-photos/")); @@ -50,6 +52,7 @@ public class ThechiveRipperTest extends RippersTest { } @Test + @Tag("flaky") public void testTheChiveGif() throws IOException { ThechiveRipper ripper = new ThechiveRipper( new URL("https://thechive.com/2019/03/14/dont-tease-me-just-squeeze-me-20-gifs/")); From d8cd6291a5c32f997e05717fa396d8164b291def Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 3 Oct 2021 18:42:59 +0200 Subject: [PATCH 222/512] set referrer in erome, fixes #47 --- .../java/com/rarchives/ripme/ripper/rippers/EromeRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java index b44d34d4..932788f2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java @@ -43,7 +43,7 @@ public class EromeRipper extends AbstractHTMLRipper { @Override public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); + addURLToDownload(url, getPrefix(index), "", "erome.com", this.cookies); } @Override From b28c4bf86abb0b889aee08d4f7bf9115671847f7 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 3 Oct 2021 19:20:22 +0200 Subject: [PATCH 223/512] gradle, display stack trace on failed test --- build.gradle.kts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index f8ef8b63..59fb0c8c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -73,6 +73,9 @@ tasks.withType { } tasks.test { + testLogging { + showStackTraces = true + } useJUnitPlatform { // gradle-6.5.1 not yet allows passing this as parameter, so exclude it excludeTags("flaky","slow") From 130a57d5459e32a66dfd3c4067296486a97802e8 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 3 Oct 2021 23:10:30 +0200 Subject: [PATCH 224/512] e62ripper, erofusripper, imagebamrupper flaky --- .../rarchives/ripme/tst/ripper/rippers/E621RipperTest.java | 6 ++++++ .../ripme/tst/ripper/rippers/ErofusRipperTest.java | 3 +++ .../ripme/tst/ripper/rippers/ImagebamRipperTest.java | 1 + 3 files changed, 10 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java index 63249423..31ce2260 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java @@ -5,6 +5,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.E621Ripper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class E621RipperTest extends RippersTest { @@ -13,11 +14,13 @@ public class E621RipperTest extends RippersTest { testRipper(ripper); } @Test + @Tag("flaky") public void testFlashOrWebm() throws IOException { E621Ripper ripper = new E621Ripper(new URL("https://e621.net/posts?page=4&tags=gif+rating%3As+3d")); testRipper(ripper); } @Test + @Tag("flaky") public void testGetNextPage() throws IOException { E621Ripper nextPageRipper = new E621Ripper(new URL("https://e621.net/posts?tags=cosmicminerals")); try { @@ -35,16 +38,19 @@ public class E621RipperTest extends RippersTest { } } @Test + @Tag("flaky") public void testOldRip() throws IOException { E621Ripper ripper = new E621Ripper(new URL("https://e621.net/post/index/1/beach")); testRipper(ripper); } @Test + @Tag("flaky") public void testOldFlashOrWebm() throws IOException { E621Ripper ripper = new E621Ripper(new URL("https://e621.net/post/index/1/gif")); testRipper(ripper); } @Test + @Tag("flaky") public void testOldGetNextPage() throws IOException { E621Ripper nextPageRipper = new E621Ripper(new URL("https://e621.net/post/index/1/cosmicminerals")); try { diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java index 6acd6e08..81192040 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java @@ -5,16 +5,19 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.ErofusRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class ErofusRipperTest extends RippersTest { @Test + @Tag("flaky") // if url does not exist, erofusripper test ends in out of memory public void testRip() throws IOException { ErofusRipper ripper = new ErofusRipper(new URL("https://www.erofus.com/comics/be-story-club-comics/a-kiss/issue-1")); testRipper(ripper); } @Test + @Tag("flaky") public void testGetGID() throws IOException { ErofusRipper ripper = new ErofusRipper(new URL("https://www.erofus.com/comics/be-story-club-comics/a-kiss/issue-1")); Assertions.assertEquals("be-story-club-comics", ripper.getGID(new URL("https://www.erofus.com/comics/be-story-club-comics/a-kiss/issue-1"))); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java index a12f93cf..5ecfe3f6 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java @@ -9,6 +9,7 @@ import org.junit.jupiter.api.Test; public class ImagebamRipperTest extends RippersTest { @Test + @Tag("flaky") public void testImagebamRip() throws IOException { ImagebamRipper ripper = new ImagebamRipper(new URL("http://www.imagebam.com/gallery/488cc796sllyf7o5srds8kpaz1t4m78i")); testRipper(ripper); From 185e591e9dff653d433acee222c5237314033ca4 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 4 Oct 2021 15:55:09 +0200 Subject: [PATCH 225/512] comicextraripper flaky tests --- .../ripme/tst/ripper/rippers/ComicextraRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java index 0769e295..75c1f955 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ComicextraRipper; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class ComicextraRipperTest extends RippersTest { @Test + @Tag("flaky") public void testComicUrl() throws IOException { URL url = new URL("https://www.comicextra.com/comic/karma-police"); ComicextraRipper ripper = new ComicextraRipper(url); From 02af0ebcec682a8abd522d59ca28c7b4505edb7a Mon Sep 17 00:00:00 2001 From: katsadim Date: Sun, 24 Oct 2021 01:02:56 +0300 Subject: [PATCH 226/512] Add el_GR translation --- .../resources/LabelsBundle_el_GR.properties | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 src/main/resources/LabelsBundle_el_GR.properties diff --git a/src/main/resources/LabelsBundle_el_GR.properties b/src/main/resources/LabelsBundle_el_GR.properties new file mode 100644 index 00000000..573e76e6 --- /dev/null +++ b/src/main/resources/LabelsBundle_el_GR.properties @@ -0,0 +1,75 @@ +Log = Log +History = Ιστορικό +created = δημιουργήθηκε +modified = τροποποιήθηκε +queue = Ουρά +Configuration = Ρυθμίσεις +open = Άνοιγμα + +# Keys for the Configuration menu +current.version = Τρέχουσα έκδοση +check.for.updates = Έλεγχος για ενημερώσεις +auto.update = Αυτόματη ενημέρωση? +max.download.threads = Μέγιστος αριθμός παράλληλων συνδέσεων: +timeout.mill = Λήξη (σε χιλιοστά του δευτερολέπτου): +retry.download.count = Αριθμός επανάληψεων μεταφόρτωσης +overwrite.existing.files = Να αντικατασταθούν τα υπάρχοντα αρχεία? +sound.when.rip.completes = Ήχος όταν το rip ολοκληρωθεί +preserve.order = Διατήρηση σειράς +save.logs = Αποθήκευση logs +notification.when.rip.starts = Ειδοποίηση όταν αρχισει το rip +save.urls.only = Αποθήκευση μόνο των URL +save.album.titles = Αποθήκευση τίτλων των αλμπουμ +autorip.from.clipboard = Αυτόματο rip-άρισμα απο το πρόχειρο (clipboard) +save.descriptions = Αποθήκευση περιγραφής +prefer.mp4.over.gif = Προτίμηση MP4 απο GIF +restore.window.position = Επαναφορά θέσης παραθύρου +remember.url.history = Να θυμάμαι ιστορικο απο URL +loading.history.from = Φόρτωση ιστορικού από + +# Queue keys +queue.remove.all = Διαγραφή όλων +queue.validation = Είσαι σίγουρος οτι θέλεις να διαγράφουν όλα τα στοιχεια της ουράς? +queue.remove.selected = Διαγραφή επιλεγμένου + +# History +re-rip.checked = Re-rip Τσεκαρισμένο +remove = Διαγραφή +clear = Καθάρισμα +history.check.all = Επιλογή όλων +history.check.none = Επιλογή κανενός +history.check.selected = Επιλογή επιλεγμένου +history.uncheck.selected = απο-επιλογή επιλεγμένου +history.load.failed.warning = Το RipMe απέτυχε να φορτώσει το αρχείο ιστορικού απο historyFile.getAbsolutePath() \n\nΛάθος %s\n\n Κλέισιμο του RipMe, θα επιφέρει αυτοματη αντικατάσταση των περιεχόμενων αυτού του αρχείου,\nάρα ίσως θα ήταν καλή ιδέα να πάρεις ένα αντίγραφο ασφαλείας πρίν κλέισεις το RipMe! +history.load.none = Δεν υπάρχουν ιστορικές εγγραφές για να ξαναripάρω. Rip-αρε καποια αλμπουμς πρώτα +history.load.none.checked = Δεν έχει τσεκαριστεί καμια ιστορική εγγραφή. Τσέκαρε μια εγγραφή κλικάροντας το checkbox στα αριστερλα του URL, ή πάτα δεξί κλίκ σε ένα URL για να επιλέξεις/αποεπιλέξεις ολα τα στοιχεία + +# TrayIcon +tray.show = Εμφάνιση +tray.hide = Απόκρυψη +tray.autorip = Αυτόματο rip του προχειρου(clipboard) +tray.exit = Έξοδος + +# Misc UI keys +loading.history.from.configuration = Φόρτωση ιστορικού απο τίς ρυθμίσεις +interrupted.while.waiting.to.rip.next.album = Διεκόπη ενω ήμουν σε αναμονή για να ripάρω το επόμενο άλμπουμ +inactive = Αδρανής +download.url.list = Μεταφόρτωση λίστας URL +select.save.dir = Επιλογή τοποθεσίας απθήκευσης + +# Keys for the logs generated by DownloadFileThread +nonretriable.status.code = Non-retriable status code +retriable.status.code = Retriable status code +server.doesnt.support.resuming.downloads = Ο διακομιστής δεν υποστηρίζει συνέχιση της μεταφόρτωσης +# A "magic number" can also be called a file signature +was.unable.to.get.content.type.using.magic.number = Δεν ευρέθη ο τύπος του αρχείου χρησιμοποιώντας τον μαγικό αριθμό +magic.number.was = Ο μαγικός αριθμός ήταν +deleting.existing.file = Διαγραφή υπάρχοντος αρχείου +request.properties = Ιδιότητες της αίτησης +download.interrupted = Η μεταφόρτωση διεκόπη +exceeded.maximum.retries = Υπέρβαση του μέγιστου αριθμου προσπαθειών +http.status.exception = HTTP status λάθος +exception.while.downloading.file = Λάθος ενω μεταφορτώνοταν ενα αρχειο +failed.to.download = Αποτυχία μεταφόρτωσης +skipping = Παράκαμψη +file.already.exists = το αρχείο υπάρχει ήδη \ No newline at end of file From 5272024647b97c5b0f6a1fea83f5d4dd05303b43 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 7 Nov 2021 07:39:37 +0100 Subject: [PATCH 227/512] ripping.interrupted does not exist, use downlowd.interrupted --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 935e48d9..dbe16245 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -725,8 +725,8 @@ public final class MainWindow implements Runnable, RipStatusHandler { statusProgress.setVisible(false); pack(); statusProgress.setValue(0); - status(Utils.getLocalizedString("ripping.interrupted")); - appendLog("Ripper interrupted", Color.RED); + status(Utils.getLocalizedString("download.interrupted")); + appendLog("Download interrupted", Color.RED); } }); optionLog.addActionListener(event -> { From f0952a3f6e620cbcd195a7d87cc049f551289239 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 12 Dec 2021 02:17:50 +0100 Subject: [PATCH 228/512] update to gradle-7.3.1 --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ffed3a25..84d1f85f 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 9496962cd018f55417d3c79b3c9117db0adc98d6 Mon Sep 17 00:00:00 2001 From: Irvin Lara Date: Sun, 19 Dec 2021 19:02:42 -0700 Subject: [PATCH 229/512] Add DanbooruRipper and tests to the ripper --- .../ripme/ripper/rippers/DanbooruRipper.java | 112 ++++++++++++++++++ .../java/com/rarchives/ripme/utils/Http.java | 7 ++ .../ripper/rippers/DanbooruRipperTest.java | 45 +++++++ 3 files changed, 164 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/DanbooruRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java new file mode 100644 index 00000000..8fdd55a1 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java @@ -0,0 +1,112 @@ +package com.rarchives.ripme.ripper.rippers; + +import com.rarchives.ripme.ripper.AbstractJSONRipper; +import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.Utils; +import org.apache.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class DanbooruRipper extends AbstractJSONRipper { + private static final Logger logger = Logger.getLogger(DanbooruRipper.class); + + private static final String DOMAIN = "danbooru.donmai.us", + HOST = "danbooru"; + + private Pattern gidPattern = null; + + private int currentPageNum = 1; + + public DanbooruRipper(URL url) throws IOException { + super(url); + } + + @Override + protected String getDomain() { + return DOMAIN; + } + + @Override + public String getHost() { + return HOST; + } + + private String getPage(int num) throws MalformedURLException { + return "https://" + getDomain() + "/posts.json?page=" + num + "&tags=" + getTag(url); + } + + @Override + protected JSONObject getFirstPage() throws IOException { + String newCompatibleJSON = "{ resources:" + Http.url(getPage(1)).getJSONArray() + " }"; + + return new JSONObject(newCompatibleJSON); + } + + @Override + protected JSONObject getNextPage(JSONObject doc) throws IOException { + currentPageNum++; + + JSONArray resourcesJSONArray = Http.url(getPage(currentPageNum)).getJSONArray(); + + int resourcesJSONArrayLength = resourcesJSONArray.length(); + + if (resourcesJSONArrayLength == 0) { + currentPageNum = 0; + throw new IOException("No more images in the next page"); + } + + String newCompatibleJSON = "{ resources:" + resourcesJSONArray + " }"; + + return new JSONObject(newCompatibleJSON); + } + + @Override + protected List getURLsFromJSON(JSONObject json) { + List res = new ArrayList<>(100); + JSONArray jsonArray = json.getJSONArray("resources"); + for (int i = 0; i < jsonArray.length(); i++) { + if (jsonArray.getJSONObject(i).has("file_url")) { + res.add(jsonArray.getJSONObject(i).getString("file_url")); + } + } + return res; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + try { + return Utils.filesystemSafe(new URI(getTag(url).replaceAll("([?&])tags=", "")).getPath()); + } catch (URISyntaxException ex) { + logger.error(ex); + } + + throw new MalformedURLException("Expected booru URL format: " + getDomain() + "/posts?tags=searchterm - got " + url + " instead"); + } + + @Override + protected void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + + private String getTag(URL url) throws MalformedURLException { + gidPattern = Pattern.compile("https?://danbooru.donmai.us/(posts)?.*([?&]tags=([a-zA-Z0-9$_.+!*'(),%-]+))(&|(#.*)?$)"); + Matcher m = gidPattern.matcher(url.toExternalForm()); + + if (m.matches()) { + return m.group(3); + } + + throw new MalformedURLException("Expected danbooru URL format: " + getDomain() + "/posts?tags=searchterm - got " + url + " instead"); + } + +} diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index fb90bbd8..d39406e7 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -9,6 +9,7 @@ import java.util.Map; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; +import org.json.JSONArray; import org.json.JSONObject; import org.jsoup.Connection; import org.jsoup.Connection.Method; @@ -171,6 +172,12 @@ public class Http { return new JSONObject(jsonString); } + public JSONArray getJSONArray() throws IOException { + ignoreContentType(); + String jsonArray = response().body(); + return new JSONArray(jsonArray); + } + public Response response() throws IOException { Response response = null; IOException lastException = null; diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DanbooruRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DanbooruRipperTest.java new file mode 100644 index 00000000..575864a5 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DanbooruRipperTest.java @@ -0,0 +1,45 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.DanbooruRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +public class DanbooruRipperTest extends RippersTest { + @Test + public void testRip() throws IOException { + List passURLs = new ArrayList<>(); + passURLs.add(new URL("https://danbooru.donmai.us/posts?tags=brown_necktie")); + passURLs.add(new URL("https://danbooru.donmai.us/posts?page=1&tags=pink_sweater_vest")); + + for (URL url : passURLs) { + DanbooruRipper danbooruRipper = new DanbooruRipper(url); + testRipper(danbooruRipper); + } + } + + @Test + public void testGetGID() throws IOException { + URL danBooruUrl = new URL("https://danbooru.donmai.us/posts?tags=brown_necktie"); + URL danBooruUrl2 = new URL("https://danbooru.donmai.us/posts?page=1&tags=pink_sweater_vest"); + + DanbooruRipper danbooruRipper = new DanbooruRipper(danBooruUrl); + DanbooruRipper danbooruRipper2 = new DanbooruRipper(danBooruUrl2); + + Assertions.assertEquals("brown_necktie", danbooruRipper.getGID(danBooruUrl)); + Assertions.assertEquals("pink_sweater_vest", danbooruRipper2.getGID(danBooruUrl2)); + } + + @Test + public void testGetHost() throws IOException { + URL danBooruUrl = new URL("https://danbooru.donmai.us/posts?tags=brown_necktie"); + + DanbooruRipper danbooruRipper = new DanbooruRipper(danBooruUrl); + + Assertions.assertEquals("danbooru", danbooruRipper.getHost()); + } +} From edc7fa4cfa404d70090fe010b20c8728bf430a28 Mon Sep 17 00:00:00 2001 From: Irvin Lara Date: Sun, 19 Dec 2021 19:11:08 -0700 Subject: [PATCH 230/512] Add tests to BooruRipper --- .../tst/ripper/rippers/BooruRipperTest.java | 47 +++++++++++++++++-- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BooruRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BooruRipperTest.java index 8fa2cfc0..f7918aad 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BooruRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BooruRipperTest.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; import java.net.URL; +import java.util.ArrayList; +import java.util.List; import com.rarchives.ripme.ripper.rippers.BooruRipper; import org.junit.jupiter.api.Assertions; @@ -10,14 +12,49 @@ import org.junit.jupiter.api.Test; public class BooruRipperTest extends RippersTest { @Test public void testRip() throws IOException { - BooruRipper ripper = new BooruRipper(new URL("http://xbooru.com/index.php?page=post&s=list&tags=furry")); - testRipper(ripper); + List passURLs = new ArrayList<>(); + passURLs.add(new URL("https://xbooru.com/index.php?page=post&s=list&tags=furry")); + passURLs.add(new URL("https://gelbooru.com/index.php?page=post&s=list&tags=animal_ears")); + + for (URL url : passURLs) { + BooruRipper ripper = new BooruRipper(url); + testRipper(ripper); + } } @Test public void testGetGID() throws IOException { - URL url = new URL("http://xbooru.com/index.php?page=post&s=list&tags=furry"); - BooruRipper ripper = new BooruRipper(url); - Assertions.assertEquals("furry", ripper.getGID(url)); + URL xbooruUrl = new URL("https://xbooru.com/index.php?page=post&s=list&tags=furry"); + URL gelbooruUrl = new URL("https://gelbooru.com/index.php?page=post&s=list&tags=animal_ears"); + + BooruRipper xbooruRipper = new BooruRipper(xbooruUrl); + BooruRipper gelbooruRipper = new BooruRipper(gelbooruUrl); + + Assertions.assertEquals("furry", xbooruRipper.getGID(xbooruUrl)); + Assertions.assertEquals("animal_ears", gelbooruRipper.getGID(gelbooruUrl)); + } + + @Test + public void testGetDomain() throws IOException { + URL xbooruUrl = new URL("https://xbooru.com/index.php?page=post&s=list&tags=furry"); + URL gelbooruUrl = new URL("https://gelbooru.com/index.php?page=post&s=list&tags=animal_ears"); + + BooruRipper xbooruRipper = new BooruRipper(xbooruUrl); + BooruRipper gelbooruRipper = new BooruRipper(gelbooruUrl); + + Assertions.assertEquals("xbooru.com", xbooruRipper.getDomain()); + Assertions.assertEquals("gelbooru.com", gelbooruRipper.getDomain()); + } + + @Test + public void testGetHost() throws IOException { + URL xbooruUrl = new URL("https://xbooru.com/index.php?page=post&s=list&tags=furry"); + URL gelbooruUrl = new URL("https://gelbooru.com/index.php?page=post&s=list&tags=animal_ears"); + + BooruRipper xbooruRipper = new BooruRipper(xbooruUrl); + BooruRipper gelbooruRipper = new BooruRipper(gelbooruUrl); + + Assertions.assertEquals("xbooru", xbooruRipper.getHost()); + Assertions.assertEquals("gelbooru", gelbooruRipper.getHost()); } } \ No newline at end of file From 3bd9d1c4127dbdfe0ef07b7a12d3a66adf57bd55 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 20 Dec 2021 02:11:06 +0100 Subject: [PATCH 231/512] update dependency versions --- build.gradle.kts | 22 +++++++++---------- gradle/wrapper/gradle-wrapper.properties | 2 +- .../java/com/rarchives/ripme/utils/Http.java | 1 - 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 59fb0c8c..68b63b30 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -11,19 +11,19 @@ repositories { } dependencies { - implementation("com.lmax:disruptor:3.4.2") - implementation("org.java-websocket:Java-WebSocket:1.5.1") - implementation("org.jsoup:jsoup:1.8.1") - implementation("org.json:json:20190722") - implementation("commons-configuration:commons-configuration:1.7") - implementation("commons-cli:commons-cli:1.4") - implementation("commons-io:commons-io:2.9.0") + implementation("com.lmax:disruptor:3.4.4") + implementation("org.java-websocket:Java-WebSocket:1.5.2") + implementation("org.jsoup:jsoup:1.14.3") + implementation("org.json:json:20211205") + implementation("commons-configuration:commons-configuration:1.10") + implementation("commons-cli:commons-cli:1.5.0") + implementation("commons-io:commons-io:2.11.0") implementation("org.apache.httpcomponents:httpclient:4.5.13") implementation("org.apache.httpcomponents:httpmime:4.5.13") - implementation("org.apache.logging.log4j:log4j-api:2.14.1") - implementation("org.apache.logging.log4j:log4j-core:2.14.1") - implementation("org.graalvm.js:js:21.1.0") - testImplementation(enforcedPlatform("org.junit:junit-bom:5.7.2")) + implementation("org.apache.logging.log4j:log4j-api:2.17.0") + implementation("org.apache.logging.log4j:log4j-core:2.17.0") + implementation("org.graalvm.js:js:21.3.0") + testImplementation(enforcedPlatform("org.junit:junit-bom:5.8.2")) testImplementation("org.junit.jupiter:junit-jupiter") testImplementation("junit:junit:4.13.2") } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 84d1f85f..d2880ba8 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index 71107e87..47ee0f29 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -14,7 +14,6 @@ import org.json.JSONObject; import org.jsoup.Connection; import org.jsoup.Connection.Method; import org.jsoup.Connection.Response; -import org.jsoup.helper.StringUtil; import org.jsoup.Jsoup; import org.jsoup.HttpStatusException; import org.jsoup.nodes.Document; From 98cc12d892988a3b327bfb3014390c6b47454f6f Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 20 Dec 2021 02:16:16 +0100 Subject: [PATCH 232/512] cleanup utils/Http.java --- .../java/com/rarchives/ripme/utils/Http.java | 69 +++++++++++-------- 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index 47ee0f29..fe020041 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -1,28 +1,26 @@ package com.rarchives.ripme.utils; +import com.rarchives.ripme.ripper.AbstractRipper; +import org.apache.commons.lang.ArrayUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONObject; +import org.jsoup.Connection; +import org.jsoup.Connection.Method; +import org.jsoup.Connection.Response; +import org.jsoup.HttpStatusException; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; + import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.HashMap; import java.util.Map; -import org.apache.commons.lang.ArrayUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.json.JSONObject; -import org.jsoup.Connection; -import org.jsoup.Connection.Method; -import org.jsoup.Connection.Response; -import org.jsoup.Jsoup; -import org.jsoup.HttpStatusException; -import org.jsoup.nodes.Document; - -import com.rarchives.ripme.ripper.AbstractRipper; - /** * Wrapper around the Jsoup connection methods. - * + *

* Benefit is retry logic. */ public class Http { @@ -31,7 +29,7 @@ public class Http { private static final Logger logger = LogManager.getLogger(Http.class); private int retries; - private String url; + private final String url; private Connection connection; // Constructors @@ -39,6 +37,7 @@ public class Http { this.url = url; defaultSettings(); } + private Http(URL url) { this.url = url.toExternalForm(); defaultSettings(); @@ -47,6 +46,7 @@ public class Http { public static Http url(String url) { return new Http(url); } + public static Http url(URL url) { return new Http(url); } @@ -68,7 +68,7 @@ public class Http { private Map cookiesForURL(String u) { Map cookiesParsed = new HashMap<>(); - String cookieDomain = ""; + String cookieDomain = ""; try { URL parsed = new URL(u); String cookieStr = ""; @@ -84,7 +84,7 @@ public class Http { logger.info("Trying to load cookies from config for " + domain); cookieStr = Utils.getConfigString("cookies." + domain, ""); if (!cookieStr.equals("")) { - cookieDomain = domain; + cookieDomain = domain; // we found something, start parsing break; } @@ -110,42 +110,52 @@ public class Http { connection.timeout(timeout); return this; } + public Http ignoreContentType() { connection.ignoreContentType(true); return this; } - public Http referrer(String ref) { + + public Http referrer(String ref) { connection.referrer(ref); return this; } + public Http referrer(URL ref) { return referrer(ref.toExternalForm()); } - public Http userAgent(String ua) { + + public Http userAgent(String ua) { connection.userAgent(ua); return this; } + public Http retries(int tries) { this.retries = tries; return this; } + public Http header(String name, String value) { - connection.header(name, value); + connection.header(name, value); return this; } - public Http cookies(Map cookies) { + + public Http cookies(Map cookies) { connection.cookies(cookies); return this; } - public Http data(Map data) { + + public Http data(Map data) { connection.data(data); return this; } + public Http data(String name, String value) { - Map data = new HashMap<>(); + Map data = new HashMap<>(); data.put(name, value); return data(data); } + public Http method(Method method) { connection.method(method); return this; @@ -155,6 +165,7 @@ public class Http { public Connection connection() { return connection; } + public Document get() throws IOException { connection.method(Method.GET); return response().parse(); @@ -172,7 +183,7 @@ public class Http { } public Response response() throws IOException { - Response response = null; + Response response; IOException lastException = null; int retries = this.retries; while (--retries >= 0) { @@ -182,15 +193,15 @@ public class Http { } catch (IOException e) { // Warn users about possibly fixable permission error if (e instanceof org.jsoup.HttpStatusException) { - HttpStatusException ex = (HttpStatusException)e; - + HttpStatusException ex = (HttpStatusException) e; + // These status codes might indicate missing cookies // 401 Unauthorized // 403 Forbidden - int status = ex.getStatusCode(); + int status = ex.getStatusCode(); if (status == 401 || status == 403) { - throw new IOException("Failed to load " + url + ": Status Code " + Integer.toString(status) + ". You might be able to circumvent this error by setting cookies for this domain" , e); + throw new IOException("Failed to load " + url + ": Status Code " + status + ". You might be able to circumvent this error by setting cookies for this domain", e); } } From 846ae6925c6b42c85a89dbace8020760a3ada82c Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 20 Dec 2021 02:28:47 +0100 Subject: [PATCH 233/512] java-17, distribution eclipse temurin --- .github/workflows/gradle.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 8fdfbebb..b83caa07 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -9,10 +9,10 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - java: [1.8] + java: [8] include: # test newest java on one os only, upload from ubuntu java8 - os: ubuntu-latest - java: 1.15 + java: 17 - os: ubuntu-latest upload: true @@ -24,9 +24,10 @@ jobs: id: ci-env uses: FranzDiebold/github-env-vars-action@v2 - - name: Set up JDK - uses: actions/setup-java@v1 + - name: Set up Java + uses: actions/setup-java@v2 with: + distribution: 'temurin' java-version: ${{ matrix.java }} - name: Cache Gradle packages From f1db4300c2a8f81737bdc17de7ba7ecabf1cb5b2 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 20 Dec 2021 03:24:36 +0100 Subject: [PATCH 234/512] mastodon, teenplane tests flaky --- .../rarchives/ripme/tst/ripper/rippers/MastodonRipperTest.java | 2 ++ .../ripme/tst/ripper/rippers/TeenplanetRipperTest.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MastodonRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MastodonRipperTest.java index 4b564300..def7525a 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MastodonRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MastodonRipperTest.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.MastodonRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class MastodonRipperTest extends RippersTest { @Test + @Tag("flaky") public void testRip() throws IOException { MastodonRipper ripper = new MastodonRipper(new URL("https://mastodon.social/@pythonhub/media")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java index 6d27ca7c..6e42b5db 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java @@ -5,10 +5,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.TeenplanetRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class TeenplanetRipperTest extends RippersTest { @Test + @Tag("flaky") public void testTeenplanetRip() throws IOException { TeenplanetRipper ripper = new TeenplanetRipper(new URL("http://teenplanet.org/galleries/the-perfect-side-of-me-6588.html")); testRipper(ripper); From 18f141bbef7d849a5aece7304f79f70c929a2072 Mon Sep 17 00:00:00 2001 From: borderline232 Date: Sat, 16 Jan 2021 01:08:05 -0500 Subject: [PATCH 235/512] Replaced fuzzyExists and Fixed regex in redgifs user and search - Added a dot in redgifs regex to account for them in usernames and search queries - Replaced fuzzyExists with better library efficient call that check the folder for existing file --- .../rarchives/ripme/ripper/DownloadFileThread.java | 3 ++- .../rarchives/ripme/ripper/rippers/RedgifsRipper.java | 8 ++++---- src/main/java/com/rarchives/ripme/utils/Utils.java | 4 ++++ .../ripme/tst/ripper/rippers/RedditRipperTest.java | 11 +++++++++++ 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 98121ed1..9b515d1d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -5,6 +5,7 @@ import java.net.HttpURLConnection; import java.net.SocketTimeoutException; import java.net.URL; import java.net.URLConnection; +import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -79,7 +80,7 @@ class DownloadFileThread extends Thread { return; } if (saveAs.exists() && !observer.tryResumeDownload() && !getFileExtFromMIME - || Utils.fuzzyExists(new File(saveAs.getParent()), saveAs.getName()) && getFileExtFromMIME + || Utils.fuzzyExistsBetter(Paths.get(saveAs.getParent()), saveAs.getName()) && getFileExtFromMIME && !observer.tryResumeDownload()) { if (Utils.getConfigBoolean("file.overwrite", false)) { logger.info("[!] " + Utils.getLocalizedString("deleting.existing.file") + prettySaveAs); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java index 17105ee4..04442abf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java @@ -55,12 +55,12 @@ public class RedgifsRipper extends AbstractHTMLRipper { } public Matcher isProfile() { - Pattern p = Pattern.compile("^https?://[wm.]*redgifs\\.com/users/([a-zA-Z0-9_-]+).*$"); + Pattern p = Pattern.compile("^https?://[wm.]*redgifs\\.com/users/([a-zA-Z0-9_.-]+).*$"); return p.matcher(url.toExternalForm()); } public Matcher isSearch() { - Pattern p = Pattern.compile("^https?://[wm.]*redgifs\\.com/gifs/browse/([a-zA-Z0-9_-]+).*$"); + Pattern p = Pattern.compile("^https?://[wm.]*redgifs\\.com/gifs/browse/([a-zA-Z0-9_.-]+).*$"); return p.matcher(url.toExternalForm()); } @@ -131,7 +131,7 @@ public class RedgifsRipper extends AbstractHTMLRipper { .ignoreContentType().get(); return (hasURLs(d).isEmpty()) ? null : d; } else { - if (cursor.equals("")) { + if (cursor.equals("") || cursor.equals("null")) { return null; } else { Document d = Http.url(new URL("https://napi.redgifs.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor)).ignoreContentType().get(); @@ -170,7 +170,7 @@ public class RedgifsRipper extends AbstractHTMLRipper { for (int i = 0; i < content.length(); i++) { result.add(content.getJSONObject(i).getString("mp4Url")); } - cursor = page.getString("cursor"); + cursor = page.get("cursor").toString(); return result; } diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 842d3e0a..947d7ef7 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -880,6 +880,10 @@ public class Utils { return false; } + public static boolean fuzzyExistsBetter(Path folder, String filename) { + return Files.exists(folder.resolve(filename)); + } + public static String sanitizeSaveAs(String fileNameToSan) { return fileNameToSan.replaceAll("[\\\\/:*?\"<>|]", "_"); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java index f4dbe327..8ad6fd09 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java @@ -60,6 +60,17 @@ public class RedditRipperTest extends RippersTest { testRipper(ripper); } + /** + * GFYCAT TEST Tests a gfycat URL with the gifdeliverynetwork/redgifs hosted video + * + * @throws IOException + */ + @Test + public void testRedditGfycatRedirectURL() throws IOException { + RedditRipper ripper = new RedditRipper( + new URL("https://www.reddit.com/r/NSFW_GIF/comments/ennwsa/gorgeous_tits/")); + } + @Test @Tag("flaky") public void testRedditGallery() throws IOException{ From 67dd4875d941799f2a333a8415c941e4e5ab0573 Mon Sep 17 00:00:00 2001 From: borderline232 Date: Sun, 8 Aug 2021 00:25:20 -0400 Subject: [PATCH 236/512] Added Reddit selfPost functionality and fixed Redgifs - Added selfpost download functionality, currently it is fixed and cannot be disabled in the jar - Fixed Redgifs using mobile mp4 in its document and instead uses its api instead to fetch the hd version --- .../ripme/ripper/rippers/RedditRipper.java | 124 ++++++++++++++++++ .../ripme/ripper/rippers/RedgifsRipper.java | 55 ++++---- .../tst/ripper/rippers/RedditRipperTest.java | 14 ++ .../tst/ripper/rippers/RedgifsRipperTest.java | 11 +- 4 files changed, 178 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 09569fc7..765f9797 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -1,14 +1,18 @@ package com.rarchives.ripme.ripper.rippers; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.util.Date; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import com.rarchives.ripme.ui.RipStatusMessage; +import j2html.TagCreator; +import j2html.tags.ContainerTag; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -19,6 +23,9 @@ import com.rarchives.ripme.ui.UpdateUtils; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; +import org.jsoup.Jsoup; + +import static j2html.TagCreator.*; public class RedditRipper extends AlbumRipper { @@ -104,6 +111,14 @@ public class RedditRipper extends AlbumRipper { children = data.getJSONArray("children"); for (int j = 0; j < children.length(); j++) { parseJsonChild(children.getJSONObject(j)); + + if (children.getJSONObject(j).getString("kind").equals("t3") && + children.getJSONObject(j).getJSONObject("data").getBoolean("is_self") + ) { + URL selfPostURL = new URL(children.getJSONObject(j).getJSONObject("data").getString("url")); + System.out.println(selfPostURL.toExternalForm()); + saveText(getJsonArrayFromURL(getJsonURL(selfPostURL))); + } } if (data.has("after") && !data.isNull("after")) { String nextURLString = Utils.stripURLParameter(url.toExternalForm(), "after"); @@ -225,6 +240,112 @@ public class RedditRipper extends AlbumRipper { } } + private void saveText(JSONArray jsonArray) throws JSONException { + File saveFileAs; + + JSONObject selfPost = jsonArray.getJSONObject(0).getJSONObject("data") + .getJSONArray("children").getJSONObject(0).getJSONObject("data"); + JSONArray comments = jsonArray.getJSONObject(1).getJSONObject("data") + .getJSONArray("children"); + + if (selfPost.getString("selftext").equals("")) { return; } + + final String title = selfPost.getString("title"); + final String id = selfPost.getString("id"); + final String author = selfPost.getString("author"); + final String creationDate = new Date((long) selfPost.getInt("created") * 1000).toString(); + final String subreddit = selfPost.getString("subreddit"); + final String selfText = selfPost.getString("selftext_html"); + final String permalink = selfPost.getString("url"); + + String html = TagCreator.html( + head( + title(title), + style(rawHtml(HTML_STYLING)) + ), + body( + div( + h1(title), + a(subreddit).withHref("https://www.reddit.com/r/" + subreddit), + a("Original").withHref(permalink), + br() + ).withClass("thing"), + div( + div( + span( + a(author).withHref("https://www.reddit.com/u/" + author) + ).withClass("author op") + ).withClass("thing oppost") + .withText(creationDate) + .with(rawHtml(Jsoup.parse(selfText).text())) + ).withClass("flex") + ).with(getComments(comments, author)), + script(rawHtml(HTML_SCRIPT)) + ).renderFormatted(); + + try { + saveFileAs = new File(workingDir.getCanonicalPath() + + "" + File.separator + + id + "_" + title.replaceAll("[\\\\/:*?\"<>|]", "") + + ".html"); + FileOutputStream out = new FileOutputStream(saveFileAs); + out.write(html.getBytes()); + out.close(); + } catch (IOException e) { + LOGGER.error("[!] Error creating save file path for description '" + url + "':", e); + return; + } + + LOGGER.debug("Downloading " + url + "'s self post to " + saveFileAs); + super.retrievingSource(permalink); + if (!saveFileAs.getParentFile().exists()) { + LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); + saveFileAs.getParentFile().mkdirs(); + } + } + + private ContainerTag getComments(JSONArray comments, String author) { + ContainerTag commentsDiv = div().withId("comments"); + + for (int i = 0; i < comments.length(); i++) { + JSONObject data = comments.getJSONObject(i).getJSONObject("data"); + + ContainerTag commentDiv = + div( + span(data.getString("author")).withClasses("author", iff(data.getString("author").equals(author), "op")), + a(new Date((long) data.getInt("created") * 1000).toString()).withHref("#" + data.getString("name")) + ).withClass("thing comment").withId(data.getString("name")) + .with(rawHtml(Jsoup.parse(data.getString("body_html")).text())); + + commentDiv = getNestedComments(data, commentDiv, author); + commentsDiv.with(commentDiv); + } + return commentsDiv; + } + + private ContainerTag getNestedComments(JSONObject data, ContainerTag parentDiv, String author) { + if (data.has("replies") && data.get("replies") instanceof JSONObject) { + for (int i = 0; i <= data.getJSONObject("replies").getJSONObject("data").getJSONArray("children").length() - 1; i++) { + JSONObject nestedComment = data.getJSONObject("replies") + .getJSONObject("data") + .getJSONArray("children") + .getJSONObject(i).getJSONObject("data"); + + ContainerTag childDiv = + div( + div( + span(nestedComment.getString("author")).withClasses("author", iff(nestedComment.getString("author").equals(author), "op")), + a(new Date((long) nestedComment.getInt("created") * 1000).toString()).withHref("#" + nestedComment.getString("name")) + ).withClass("comment").withId(nestedComment.getString("name")) + .with(rawHtml(Jsoup.parse(nestedComment.getString("body_html")).text())) + ).withClass("child"); + + parentDiv.with(getNestedComments(nestedComment, childDiv, author)); + } + } + return parentDiv; + } + private URL parseRedditVideoMPD(String vidURL) { org.jsoup.nodes.Document doc = null; try { @@ -369,4 +490,7 @@ public class RedditRipper extends AlbumRipper { throw new MalformedURLException("Only accepts user pages, subreddits, post, or gallery can't understand " + url); } + private static final String HTML_STYLING = " .author { font-weight: bold; } .op { color: blue; } .comment { border: 0px; margin: 0 0 25px; padding-left: 5px; } .child { margin: 2px 0 0 20px; border-left: 2px dashed #AAF; } .collapsed { background: darkgrey; margin-bottom: 0; } .collapsed > div { display: none; } .md { max-width: 840px; padding-right: 1em; } h1 { margin: 0; } body { position: relative; background-color: #eeeeec; color: #00000a; font-weight: 400; font-style: normal; font-variant: normal; font-family: Helvetica,Arial,sans-serif; line-height: 1.4 } blockquote { margin: 5px 5px 5px 15px; padding: 1px 1px 1px 15px; max-width: 60em; border: 1px solid #ccc; border-width: 0 0 0 1px; } pre { white-space: pre-wrap; } img, video { max-width: 60vw; max-height: 90vh; object-fit: contain; } .thing { overflow: hidden; margin: 0 5px 3px 40px; border: 1px solid #e0e0e0; background-color: #fcfcfb; } :target > .md { border: 5px solid blue; } .post { margin-bottom: 20px; margin-top: 20px; } .gold { background: goldenrod; } .silver { background: silver; } .platinum { background: aqua; } .deleted { background: #faa; } .md.deleted { background: inherit; border: 5px solid #faa; } .oppost { background-color: #EEF; } blockquote > p { margin: 0; } #related { max-height: 20em; overflow-y: scroll; background-color: #F4FFF4; } #related h3 { position: sticky; top: 0; background-color: white; } .flex { display: flex; flex-flow: wrap; flex-direction: row-reverse; justify-content: flex-end; } "; + private static final String HTML_SCRIPT = "document.addEventListener('mousedown', function(e) { var t = e.target; if (t.className == 'author') { t = t.parentElement; } if (t.classList.contains('comment')) { t.classList.toggle('collapsed'); e.preventDefault(); e.stopPropagation(); return false; } });"; + } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java index 04442abf..2b169ae3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java @@ -4,6 +4,7 @@ import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; import org.json.JSONArray; import org.json.JSONObject; +import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -72,14 +73,15 @@ public class RedgifsRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { if (!isProfile().matches() && !isSearch().matches()) { - return Http.url(url).get(); + return Jsoup.connect(getJsonURL(url).toExternalForm()) + .ignoreContentType(true).get(); } else if (isSearch().matches()) { searchText = getGID(url).replace("-", " "); return Http.url( - new URL("https://napi.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart*searchCount)).ignoreContentType().get(); + new URL("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart*searchCount)).ignoreContentType().get(); } else { username = getGID(url); - return Http.url(new URL("https://napi.redgifs.com/v1/users/" + username + "/gfycats?count=" + count)) + return Http.url(new URL("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count)) .ignoreContentType().get(); } } @@ -126,7 +128,7 @@ public class RedgifsRipper extends AbstractHTMLRipper { public Document getNextPage(Document doc) throws IOException { if (isSearch().matches()) { Document d = Http.url( - new URL("https://napi.redgifs.com/v1/gfycats/search?search_text=" + searchText + new URL("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchCount*++searchStart)) .ignoreContentType().get(); return (hasURLs(d).isEmpty()) ? null : d; @@ -134,7 +136,7 @@ public class RedgifsRipper extends AbstractHTMLRipper { if (cursor.equals("") || cursor.equals("null")) { return null; } else { - Document d = Http.url(new URL("https://napi.redgifs.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor)).ignoreContentType().get(); + Document d = Http.url(new URL("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor)).ignoreContentType().get(); return (hasURLs(d).isEmpty()) ? null : d; } } @@ -146,14 +148,9 @@ public class RedgifsRipper extends AbstractHTMLRipper { if (isProfile().matches() || isSearch().matches()) { result = hasURLs(doc); } else { - Elements videos = doc.select("script"); - for (Element el : videos) { - String json = el.html(); - if (json.startsWith("{")) { - JSONObject page = new JSONObject(json); - result.add(page.getJSONObject("video").getString("contentUrl")); - } - } + JSONObject api = new JSONObject(doc.body().html()); + result.add(api.getJSONObject("gfyItem").getString("mp4Url")); + } return result; } @@ -183,19 +180,29 @@ public class RedgifsRipper extends AbstractHTMLRipper { public static String getVideoURL(URL url) throws IOException { LOGGER.info("Retrieving " + url.toExternalForm()); - //Sanitize the URL first - url = new URL(url.toExternalForm().replace("/gifs/detail", "")); + try { + Document doc = Jsoup.connect(getJsonURL(url).toExternalForm()) + .ignoreContentType(true).get(); - Document doc = Http.url(url).get(); - Elements videos = doc.select("script"); - for (Element el : videos) { - String json = el.html(); - if (json.startsWith("{")) { - JSONObject page = new JSONObject(json); - return page.getJSONObject("video").getString("contentUrl"); - } + JSONObject api = new JSONObject(doc.body().html()); + return api.getJSONObject("gfyItem").getJSONObject("content_urls") + .getJSONObject("mp4").getString("url"); + + } catch (NullPointerException e) { + return null; } - throw new IOException(); } + public static URL getJsonURL(URL url) throws MalformedURLException{ + String regex = "^https?://[wm.]*redgifs\\.com/watch/([a-zA-Z0-9_]+).*$"; + + final Pattern pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE); + final Matcher matcher = pattern.matcher(url.toExternalForm()); + + if (matcher.matches()) { + return new URL("https://api.redgifs.com/v1/gfycats/" + matcher.group(1)); + } + + return null; + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java index 8ad6fd09..20824fda 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java @@ -47,6 +47,20 @@ public class RedditRipperTest extends RippersTest { testRipper(ripper); } + @Test + public void testSelfPostRip() throws IOException { + RedditRipper ripper = new RedditRipper( + new URL("https://www.reddit.com/r/gonewildstories/comments/oz7d97/f_18_finally_having_a_normal_sex_life/") + ); + testRipper(ripper); + } + + @Test + public void testSelfPostAuthorRip() throws IOException { + RedditRipper ripper = new RedditRipper(new URL("https://www.reddit.com/user/ickybabie_")); + testRipper(ripper); + } + /** * GFYCAT TEST Tests a Bad URL with the "/gifs/detail" inside. * diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java index ed71128d..01c7a622 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java @@ -1,5 +1,6 @@ package com.rarchives.ripme.tst.ripper.rippers; +import com.rarchives.ripme.ripper.rippers.RedditRipper; import com.rarchives.ripme.ripper.rippers.RedgifsRipper; import org.jsoup.nodes.Document; import org.junit.jupiter.api.*; @@ -53,8 +54,14 @@ public class RedgifsRipperTest extends RippersTest { Document doc = ripper.getFirstPage(); doc = ripper.getNextPage(doc); - Assertions.assertTrue("https://napi.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=150".equalsIgnoreCase(doc.location())); + Assertions.assertTrue("https://api.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=150".equalsIgnoreCase(doc.location())); doc = ripper.getNextPage(doc); - Assertions.assertTrue("https://napi.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=300".equalsIgnoreCase(doc.location())); + Assertions.assertTrue("https://api.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=300".equalsIgnoreCase(doc.location())); + } + + @Test + public void testRedditRedgifs() throws IOException { + RedditRipper ripper = new RedditRipper(new URL("https://www.reddit.com/r/nsfwhardcore/comments/ouz5bw/me_cumming_on_his_face/")); + testRipper(ripper); } } From 9ca49a2fc653a44652f8b7ca430b3c0156a28c62 Mon Sep 17 00:00:00 2001 From: borderline232 Date: Sun, 8 Aug 2021 13:40:14 -0400 Subject: [PATCH 237/512] Fix: Reverted redgifs ripper to previous implementation - redgifs api blocking api calls so just replaced mobile string --- .../ripme/ripper/rippers/RedgifsRipper.java | 49 +++++++++---------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java index 2b169ae3..3c8547a9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java @@ -73,8 +73,7 @@ public class RedgifsRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { if (!isProfile().matches() && !isSearch().matches()) { - return Jsoup.connect(getJsonURL(url).toExternalForm()) - .ignoreContentType(true).get(); + return Http.url(url).get(); } else if (isSearch().matches()) { searchText = getGID(url).replace("-", " "); return Http.url( @@ -148,9 +147,15 @@ public class RedgifsRipper extends AbstractHTMLRipper { if (isProfile().matches() || isSearch().matches()) { result = hasURLs(doc); } else { - JSONObject api = new JSONObject(doc.body().html()); - result.add(api.getJSONObject("gfyItem").getString("mp4Url")); - + Elements videos = doc.select("script"); + for (Element el : videos) { + String json = el.html(); + if (json.startsWith("{")) { + JSONObject page = new JSONObject(json); + result.add(page.getJSONObject("video").getString("contentUrl") + .replace("-mobile", "")); + } + } } return result; } @@ -180,29 +185,19 @@ public class RedgifsRipper extends AbstractHTMLRipper { public static String getVideoURL(URL url) throws IOException { LOGGER.info("Retrieving " + url.toExternalForm()); - try { - Document doc = Jsoup.connect(getJsonURL(url).toExternalForm()) - .ignoreContentType(true).get(); + //Sanitize the URL first + url = new URL(url.toExternalForm().replace("/gifs/detail", "")); - JSONObject api = new JSONObject(doc.body().html()); - return api.getJSONObject("gfyItem").getJSONObject("content_urls") - .getJSONObject("mp4").getString("url"); - - } catch (NullPointerException e) { - return null; + Document doc = Http.url(url).get(); + Elements videos = doc.select("script"); + for (Element el : videos) { + String json = el.html(); + if (json.startsWith("{")) { + JSONObject page = new JSONObject(json); + String mobileUrl = page.getJSONObject("video").getString("contentUrl"); + return mobileUrl.replace("-mobile", ""); + } } - } - - public static URL getJsonURL(URL url) throws MalformedURLException{ - String regex = "^https?://[wm.]*redgifs\\.com/watch/([a-zA-Z0-9_]+).*$"; - - final Pattern pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE); - final Matcher matcher = pattern.matcher(url.toExternalForm()); - - if (matcher.matches()) { - return new URL("https://api.redgifs.com/v1/gfycats/" + matcher.group(1)); - } - - return null; + throw new IOException(); } } From 2b4d4cc1245475795f6c41d2701afba0b450ac54 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 20 Dec 2021 05:02:59 +0100 Subject: [PATCH 238/512] +j2html in build.gradle.kts, testSelfPostAuthorRip flaky --- build.gradle.kts | 1 + .../com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java | 1 + 2 files changed, 2 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index 68b63b30..78f6b490 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -15,6 +15,7 @@ dependencies { implementation("org.java-websocket:Java-WebSocket:1.5.2") implementation("org.jsoup:jsoup:1.14.3") implementation("org.json:json:20211205") + implementation("com.j2html:j2html:1.5.0") implementation("commons-configuration:commons-configuration:1.10") implementation("commons-cli:commons-cli:1.5.0") implementation("commons-io:commons-io:2.11.0") diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java index 20824fda..1641430d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java @@ -56,6 +56,7 @@ public class RedditRipperTest extends RippersTest { } @Test + @Tag("flaky") public void testSelfPostAuthorRip() throws IOException { RedditRipper ripper = new RedditRipper(new URL("https://www.reddit.com/user/ickybabie_")); testRipper(ripper); From 914ea1cb409fd2b5c076407adf0cf29e1d9eae4b Mon Sep 17 00:00:00 2001 From: Amerigo Date: Sun, 26 Dec 2021 11:34:13 +0100 Subject: [PATCH 239/512] Added referer to avoid 405 error in downloads --- .../java/com/rarchives/ripme/ripper/rippers/EromeRipper.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java index 9b586b9a..7f056dc2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java @@ -22,6 +22,8 @@ import com.rarchives.ripme.utils.Http; */ public class EromeRipper extends AbstractHTMLRipper { + private static final String EROME_REFERER = "https://www.erome.com/"; + boolean rippingProfile; @@ -41,7 +43,7 @@ public class EromeRipper extends AbstractHTMLRipper { @Override public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); + addURLToDownload(url, getPrefix(index), "", EROME_REFERER, null, null); } @Override From c653c7f0162a920d914521ea1c856bbf2442dc02 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 26 Dec 2021 13:15:29 +0100 Subject: [PATCH 240/512] add ripme-2.0.0 sha256sum --- ripme.json | 5 +++-- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ripme.json b/ripme.json index dea957c0..dfba674c 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "currentHash": "008201e406f401b27248277a4188f26203bb9da0170872de900125f8a6c8b558", + "currentHash": "8f51271e5a9e138a9d7f7febe11809619a28e7272cb231fed9b4a6faef88e8b6", "changeList": [ + "2.0.0: Fixed Zizki, WordpressComics, Imagebam; marked some tests as flaky ", "1.7.95: Added porncomixinfo.net; Fixed ripper for HentaiNexus; move l option to before r and R; marked some tests as flaky ", "1.7.94: Added reddit gallery support; Fixed AllporncomicRipper; Fix imagefap ripper; instagramRipper, replaced Nashorn with GraalVM.js", "1.7.93: Fixed Motherless ripper; Fixed e621 ripper; Updated pt_PT translation; Implemented redgifs Ripper; added missing translation to Korean/KR; Fixed elecx ripper; Added ripper for HentaiNexus", @@ -268,4 +269,4 @@ "1.0.1: Added auto-update functionality" ], "latestVersion": "1.7.95" -} \ No newline at end of file +} diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index b833a62a..6eb777db 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -26,8 +26,8 @@ public class UpdateUtils { private static final Logger logger = LogManager.getLogger(UpdateUtils.class); // do not update the default version without adjusting the unit test. the real version comes from METAINF.MF private static final String DEFAULT_VERSION = "1.7.94-10-b6345398"; - private static final String REPO_NAME = "ripmeapp/ripme"; - private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json"; + private static final String REPO_NAME = "ripmeapp2/ripme"; + private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/main/ripme.json"; private static String mainFileName; static { @@ -44,7 +44,7 @@ public class UpdateUtils { private static JSONObject ripmeJson; private static String getUpdateJarURL(String latestVersion) { - return "https://github.com/" + REPO_NAME + "/releases/download/" + latestVersion + "/ripme.jar"; + return "https://github.com/" + REPO_NAME + "/releases/download/latest-" + latestVersion + "/ripme"+ latestVersion + ".jar"; } public static String getThisJarVersion() { From 26aaa2a9c3e0e42b7b34e72d2176a3cdde568ed3 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 26 Dec 2021 13:15:29 +0100 Subject: [PATCH 241/512] add ripme-2.0.1 sha256sum --- ripme.json | 3 ++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ripme.json b/ripme.json index dfba674c..1b0a947f 100644 --- a/ripme.json +++ b/ripme.json @@ -1,6 +1,7 @@ { - "currentHash": "8f51271e5a9e138a9d7f7febe11809619a28e7272cb231fed9b4a6faef88e8b6", + "currentHash": "9eb4d095293653898eeecdf8cbf033a9c6a37c70e8da38a0f07029d86aa15745", "changeList": [ + "2.0.1: Fixed reddit, tujigu, xhamster, imagebam; marked some tests as flaky.", "2.0.0: Fixed Zizki, WordpressComics, Imagebam; marked some tests as flaky ", "1.7.95: Added porncomixinfo.net; Fixed ripper for HentaiNexus; move l option to before r and R; marked some tests as flaky ", "1.7.94: Added reddit gallery support; Fixed AllporncomicRipper; Fix imagefap ripper; instagramRipper, replaced Nashorn with GraalVM.js", diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 6eb777db..8ac26f98 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -44,7 +44,7 @@ public class UpdateUtils { private static JSONObject ripmeJson; private static String getUpdateJarURL(String latestVersion) { - return "https://github.com/" + REPO_NAME + "/releases/download/latest-" + latestVersion + "/ripme"+ latestVersion + ".jar"; + return "https://github.com/" + REPO_NAME + "/releases/download/latest/ripme"+ latestVersion + ".jar"; } public static String getThisJarVersion() { From 11fc8e25f27dc95054fbaf75fc46109795069f4f Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 26 Dec 2021 13:39:17 +0100 Subject: [PATCH 242/512] add ripme-2.0.2 sha256sum --- ripme.json | 9 +++++---- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ripme.json b/ripme.json index 1b0a947f..b38e2f21 100644 --- a/ripme.json +++ b/ripme.json @@ -1,7 +1,9 @@ { - "currentHash": "9eb4d095293653898eeecdf8cbf033a9c6a37c70e8da38a0f07029d86aa15745", + "latestVersion": "2.0.2", + "currentHash": "0e543535f274b60c2d391163f8a53344293a61e118bcf840819facce5ca7e714", "changeList": [ - "2.0.1: Fixed reddit, tujigu, xhamster, imagebam; marked some tests as flaky.", + "2.0.2: Add greek translation, fixed reddit, redgif.", + "2.0.1: Fixed reddit, tujigu, xhamster, imagebam, erome; marked some tests as flaky.", "2.0.0: Fixed Zizki, WordpressComics, Imagebam; marked some tests as flaky ", "1.7.95: Added porncomixinfo.net; Fixed ripper for HentaiNexus; move l option to before r and R; marked some tests as flaky ", "1.7.94: Added reddit gallery support; Fixed AllporncomicRipper; Fix imagefap ripper; instagramRipper, replaced Nashorn with GraalVM.js", @@ -268,6 +270,5 @@ "1.0.4: Fixed spaces-in-directory bug", "1.0.3: Added VK.com ripper", "1.0.1: Added auto-update functionality" - ], - "latestVersion": "1.7.95" + ] } diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 8ac26f98..af05d73c 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -44,7 +44,7 @@ public class UpdateUtils { private static JSONObject ripmeJson; private static String getUpdateJarURL(String latestVersion) { - return "https://github.com/" + REPO_NAME + "/releases/download/latest/ripme"+ latestVersion + ".jar"; + return "https://github.com/" + REPO_NAME + "/releases/download/"+ latestVersion + "/ripme-"+ latestVersion + ".jar"; } public static String getThisJarVersion() { From 101a6088b8f9a928a1c7a3aad633431065b91bce Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 26 Dec 2021 14:27:05 +0100 Subject: [PATCH 243/512] update to gradle-7.3.3 --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index d2880ba8..2e6e5897 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 73d37456bb8b1995fd7800d8e56396ee2a8f909c Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 26 Dec 2021 15:08:23 +0100 Subject: [PATCH 244/512] intellij reformat UpdateUtils --- .../com/rarchives/ripme/ui/UpdateUtils.java | 55 +++++++++---------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index af05d73c..b593d6ec 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -1,16 +1,6 @@ package com.rarchives.ripme.ui; -import java.awt.Dimension; -import java.io.*; -import java.net.URISyntaxException; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; - -import javax.swing.JEditorPane; -import javax.swing.JLabel; -import javax.swing.JOptionPane; -import javax.swing.JScrollPane; - +import com.rarchives.ripme.utils.Utils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.json.JSONArray; @@ -19,7 +9,19 @@ import org.jsoup.Connection.Response; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; -import com.rarchives.ripme.utils.Utils; +import javax.swing.*; +import java.awt.*; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.net.URISyntaxException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; public class UpdateUtils { @@ -28,7 +30,9 @@ public class UpdateUtils { private static final String DEFAULT_VERSION = "1.7.94-10-b6345398"; private static final String REPO_NAME = "ripmeapp2/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/main/ripme.json"; + private static final String updateFileName = "ripme.jar.update"; private static String mainFileName; + private static JSONObject ripmeJson; static { try { @@ -40,11 +44,8 @@ public class UpdateUtils { } } - private static final String updateFileName = "ripme.jar.update"; - private static JSONObject ripmeJson; - private static String getUpdateJarURL(String latestVersion) { - return "https://github.com/" + REPO_NAME + "/releases/download/"+ latestVersion + "/ripme-"+ latestVersion + ".jar"; + return "https://github.com/" + REPO_NAME + "/releases/download/" + latestVersion + "/ripme-" + latestVersion + ".jar"; } public static String getThisJarVersion() { @@ -72,7 +73,7 @@ public class UpdateUtils { public static void updateProgramCLI() { logger.info("Checking for update..."); - Document doc = null; + Document doc; try { logger.debug("Retrieving " + UpdateUtils.updateJsonURL); doc = Jsoup.connect(UpdateUtils.updateJsonURL).timeout(10 * 1000).ignoreContentType(true).get(); @@ -113,7 +114,7 @@ public class UpdateUtils { public static void updateProgramGUI(JLabel configUpdateLabel) { configUpdateLabel.setText("Checking for update..."); - Document doc = null; + Document doc; try { logger.debug("Retrieving " + UpdateUtils.updateJsonURL); doc = Jsoup.connect(UpdateUtils.updateJsonURL).timeout(10 * 1000).ignoreContentType(true).get(); @@ -199,11 +200,11 @@ public class UpdateUtils { // a version string looks like 1.7.94, 1.7.94-10-something // 10 is the number of commits since the 1.7.94 tag, so newer // the int array returned then contains e.g. 1.7.94.0 or 1.7.94.10 - String[] strVersions = version.split("[\\.-]"); + String[] strVersions = version.split("[.-]"); // not consider more than 4 components of version, loop only the real number // of components or maximum 4 components of the version string int[] intVersions = new int[4]; - for (int i = 0; i < Math.min(4,strVersions.length); i++) { + for (int i = 0; i < Math.min(4, strVersions.length); i++) { // if it is an integer, set it, otherwise leave default 0 if (strVersions[i].matches("\\d+")) { intVersions[i] = Integer.parseInt(strVersions[i]); @@ -234,11 +235,9 @@ public class UpdateUtils { // As patch.py writes the hash in lowercase this must return the has in // lowercase return sb.toString().toLowerCase(); - } catch (NoSuchAlgorithmException e) { - logger.error("Got error getting file hash " + e.getMessage()); } catch (FileNotFoundException e) { logger.error("Could not find file: " + file.getName()); - } catch (IOException e) { + } catch (NoSuchAlgorithmException | IOException e) { logger.error("Got error getting file hash " + e.getMessage()); } return null; @@ -273,15 +272,15 @@ public class UpdateUtils { // Windows final String batchFile = "update_ripme.bat"; final String batchPath = new File(batchFile).getAbsolutePath(); - String script = "@echo off\r\n" + "timeout 1\r\n" + String script = "@echo off\r\n" + "timeout 1\r\n" + "copy \"" + updateFileName + "\" \"" + mainFileName + "\"\r\n" + "del \"" + updateFileName + "\"\r\n"; - - if (shouldLaunch) + + if (shouldLaunch) script += "\"" + mainFileName + "\"\r\n"; script += "del \"" + batchPath + "\"\r\n"; - - final String[] batchExec = new String[] { batchPath }; + + final String[] batchExec = new String[]{batchPath}; // Create updater script try (BufferedWriter bw = new BufferedWriter(new FileWriter(batchFile))) { bw.write(script); From b605fdd35d1a3b9ae1536588160ec9457c40e96d Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 26 Dec 2021 15:20:13 +0100 Subject: [PATCH 245/512] vscoripper test flaky. --- .../com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java index fc78ec2d..44463c76 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java @@ -3,6 +3,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.VscoRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -28,6 +29,7 @@ public class VscoRipperTest extends RippersTest { * @throws IOException */ @Test + @Tag("flaky") public void testHyphenatedRip() throws IOException { VscoRipper ripper = new VscoRipper(new URL("https://vsco.co/jolly-roger/gallery")); testRipper(ripper); From 41b71e398e0608fb36ec4adb25610e92379f069e Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 26 Dec 2021 23:11:29 +0100 Subject: [PATCH 246/512] add ripme-2.0.3 sha256sum --- ripme.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ripme.json b/ripme.json index b38e2f21..3566b498 100644 --- a/ripme.json +++ b/ripme.json @@ -1,7 +1,8 @@ { - "latestVersion": "2.0.2", - "currentHash": "0e543535f274b60c2d391163f8a53344293a61e118bcf840819facce5ca7e714", + "latestVersion": "2.0.3", + "currentHash": "fe2e5ddec836420329a9348ccf5b019087fc750128fe485479e8b7ccee75fc93", "changeList": [ + "2.0.3: Check new version against ripme2app.", "2.0.2: Add greek translation, fixed reddit, redgif.", "2.0.1: Fixed reddit, tujigu, xhamster, imagebam, erome; marked some tests as flaky.", "2.0.0: Fixed Zizki, WordpressComics, Imagebam; marked some tests as flaky ", From 8ab3168541d79edbfccaa1246aca1187efe699db Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 27 Dec 2021 10:05:31 +0100 Subject: [PATCH 247/512] reformat Utils.java --- .../java/com/rarchives/ripme/utils/Utils.java | 63 +++++++++---------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 947d7ef7..e4337e8f 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -1,5 +1,22 @@ package com.rarchives.ripme.utils; +import com.rarchives.ripme.ripper.AbstractRipper; +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.RollingFileAppender; +import org.apache.logging.log4j.core.appender.rolling.DefaultRolloverStrategy; +import org.apache.logging.log4j.core.appender.rolling.SizeBasedTriggeringPolicy; +import org.apache.logging.log4j.core.appender.rolling.TriggeringPolicy; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; + +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.Clip; +import javax.sound.sampled.Line; +import javax.sound.sampled.LineEvent; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -24,31 +41,13 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.MissingResourceException; +import java.util.Objects; import java.util.ResourceBundle; import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.regex.Matcher; import java.util.regex.Pattern; -import javax.sound.sampled.AudioSystem; -import javax.sound.sampled.Clip; -import javax.sound.sampled.Line; -import javax.sound.sampled.LineEvent; - -import com.rarchives.ripme.ripper.AbstractRipper; - -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.core.LoggerContext; -import org.apache.logging.log4j.core.appender.RollingFileAppender; -import org.apache.logging.log4j.core.appender.rolling.DefaultRolloverStrategy; -import org.apache.logging.log4j.core.appender.rolling.SizeBasedTriggeringPolicy; -import org.apache.logging.log4j.core.appender.rolling.TriggeringPolicy; -import org.apache.logging.log4j.core.config.Configuration; -import org.apache.logging.log4j.core.config.LoggerConfig; - /** * Common utility functions used in various places throughout the project. */ @@ -64,9 +63,9 @@ public class Utils { private static PropertiesConfiguration config; private static HashMap> cookieCache; - private static HashMap magicHash = new HashMap<>(); + private static final HashMap magicHash = new HashMap<>(); - private static ResourceBundle resourceBundle = null; + private static ResourceBundle resourceBundle; static { cookieCache = new HashMap<>(); @@ -242,7 +241,7 @@ public class Utils { } private static File getJarDirectory() { - File jarDirectory = Utils.class.getResource("/rip.properties").toString().contains("jar:") + File jarDirectory = Objects.requireNonNull(Utils.class.getResource("/rip.properties")).toString().contains("jar:") ? new File(System.getProperty("java.class.path")).getParentFile() : new File(System.getProperty("user.dir")); @@ -412,6 +411,7 @@ public class Utils { if (directory != null && directory.exists()) { // Get the list of the files contained in the package String[] files = directory.list(); + assert files != null; for (String file : files) { if (file.endsWith(".class") && !file.contains("$")) { String className = pkgname + '.' + file.substring(0, file.length() - 6); @@ -524,10 +524,10 @@ public class Utils { // Get a List of all Directories and check its lowercase // if file exists return it File file = new File(path.substring(0, index)); - if (! (file.isDirectory() && file.canWrite() && file.canExecute())) { + if (!(file.isDirectory() && file.canWrite() && file.canExecute())) { throw new IOException("Original directory \"" + file + "\" is no directory or not writeable."); } - ArrayList names = new ArrayList<>(Arrays.asList(file.list())); + ArrayList names = new ArrayList<>(Arrays.asList(Objects.requireNonNull(file.list()))); for (String name : names) { if (name.toLowerCase().equals(lastPart)) { @@ -548,7 +548,7 @@ public class Utils { */ public static String bytesToHumanReadable(int bytes) { float fbytes = (float) bytes; - String[] mags = new String[] { "", "K", "M", "G", "T" }; + String[] mags = new String[]{"", "K", "M", "G", "T"}; int magIndex = 0; while (fbytes >= 1024) { fbytes /= 1024; @@ -755,7 +755,7 @@ public class Utils { * of the UI. * * @return Returns the default resource bundle using the language specified in - * the config file. + * the config file. */ public static ResourceBundle getResourceBundle(String langSelect) { if (langSelect == null) { @@ -791,7 +791,7 @@ public class Utils { public static String[] getSupportedLanguages() { ArrayList filesList = new ArrayList<>(); try { - URI uri = Utils.class.getResource("/rip.properties").toURI(); + URI uri = Objects.requireNonNull(Utils.class.getResource("/rip.properties")).toURI(); Path myPath; if (uri.getScheme().equals("jar")) { @@ -815,7 +815,7 @@ public class Utils { } catch (Exception e) { e.printStackTrace(); // On error return default language - return new String[] { DEFAULT_LANG }; + return new String[]{DEFAULT_LANG}; } } @@ -833,8 +833,7 @@ public class Utils { * @param bytesCompleted How many bytes have been downloaded * @param bytesTotal The total size of the file that is being * downloaded - * @return Returns the formatted status text for rippers using the byte progress - * bar + * @return Returns the formatted status text for rippers using the byte progresbar */ public static String getByteStatusText(int completionPercentage, int bytesCompleted, int bytesTotal) { return completionPercentage + "% - " + Utils.bytesToHumanReadable(bytesCompleted) + " / " @@ -854,8 +853,8 @@ public class Utils { } private static void initialiseMagicHashMap() { - magicHash.put(ByteBuffer.wrap(new byte[] { -1, -40, -1, -37, 0, 0, 0, 0 }), "jpeg"); - magicHash.put(ByteBuffer.wrap(new byte[] { -119, 80, 78, 71, 13, 0, 0, 0 }), "png"); + magicHash.put(ByteBuffer.wrap(new byte[]{-1, -40, -1, -37, 0, 0, 0, 0}), "jpeg"); + magicHash.put(ByteBuffer.wrap(new byte[]{-119, 80, 78, 71, 13, 0, 0, 0}), "png"); } // Checks if a file exists ignoring it's extension. From ae73b7348bdef86ae411bdb09bd3ef548a4d56e4 Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 30 Dec 2021 00:36:19 +0100 Subject: [PATCH 248/512] luscios, old --> legacy. fixes https://github.com/RipMeApp/ripme/issues/1924, but the unit tests still fail, so the ripper most likely is not fixed. --- .../ripme/ripper/rippers/LusciousRipper.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java index 7eabfc6f..e5b12df1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java @@ -19,7 +19,7 @@ import com.rarchives.ripme.utils.Http; public class LusciousRipper extends AbstractHTMLRipper { private static final int RETRY_COUNT = 5; // Keeping it high for read timeout exception. - private static final Pattern P = Pattern.compile("^https?:\\/\\/(?:members\\.|old\\.|www\\.)?luscious.net\\/albums\\/([-_.0-9a-zA-Z]+)\\/?"); + private static final Pattern P = Pattern.compile("^https?:\\/\\/(?:members\\.|legacy\\.|www\\.)?luscious.net\\/albums\\/([-_.0-9a-zA-Z]+)\\/?"); private DownloadThreadPool lusciousThreadPool = new DownloadThreadPool("lusciousThreadPool"); public LusciousRipper(URL url) throws IOException { @@ -89,15 +89,15 @@ public class LusciousRipper extends AbstractHTMLRipper { @Override public URL sanitizeURL(URL url) throws MalformedURLException { - // Sanitizes the url removing GET parameters and convert to old api url. - // "https://old.luscious.net/albums/albumname" + // Sanitizes the url removing GET parameters and convert to legacy api url. + // "https://legacy.luscious.net/albums/albumname" try { Matcher m = P.matcher(url.toString()); if (m.matches()) { String sanitizedUrl = m.group(); sanitizedUrl = sanitizedUrl.replaceFirst( - "^https?:\\/\\/(?:members\\.|old\\.|www\\.)?luscious.net", - "https://old.luscious.net"); + "^https?:\\/\\/(?:members\\.|legacy\\.|www\\.)?luscious.net", + "https://legacy.luscious.net"); return new URL(sanitizedUrl); } @@ -113,7 +113,7 @@ public class LusciousRipper extends AbstractHTMLRipper { public String normalizeUrl(String url) { try { return url.toString().replaceFirst( - "^https?:\\/\\/(?:members\\.|old\\.)?luscious.net", "https://www.luscious.net"); + "^https?:\\/\\/(?:members\\.|legacy\\.)?luscious.net", "https://www.luscious.net"); } catch (Exception e) { LOGGER.info("Error normalizing the url."); LOGGER.error(e); From 1c85cc89c52286416d9d7897b0d1411e8157b0d4 Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 30 Dec 2021 00:40:44 +0100 Subject: [PATCH 249/512] luscios, code cleanup --- .../ripme/ripper/rippers/LusciousRipper.java | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java index e5b12df1..441fc10f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java @@ -1,5 +1,12 @@ package com.rarchives.ripme.ripper.rippers; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.ripper.DownloadThreadPool; +import com.rarchives.ripme.utils.Http; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -8,19 +15,11 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; -import com.rarchives.ripme.utils.Http; - public class LusciousRipper extends AbstractHTMLRipper { private static final int RETRY_COUNT = 5; // Keeping it high for read timeout exception. - private static final Pattern P = Pattern.compile("^https?:\\/\\/(?:members\\.|legacy\\.|www\\.)?luscious.net\\/albums\\/([-_.0-9a-zA-Z]+)\\/?"); - private DownloadThreadPool lusciousThreadPool = new DownloadThreadPool("lusciousThreadPool"); + private static final Pattern P = Pattern.compile("^https?://(?:members\\.|legacy\\.|www\\.)?luscious.net/albums/([-_.0-9a-zA-Z]+)/?"); + private final DownloadThreadPool lusciousThreadPool = new DownloadThreadPool("lusciousThreadPool"); public LusciousRipper(URL url) throws IOException { super(url); @@ -96,7 +95,7 @@ public class LusciousRipper extends AbstractHTMLRipper { if (m.matches()) { String sanitizedUrl = m.group(); sanitizedUrl = sanitizedUrl.replaceFirst( - "^https?:\\/\\/(?:members\\.|legacy\\.|www\\.)?luscious.net", + "^https?://(?:members\\.|legacy\\.|www\\.)?luscious.net", "https://legacy.luscious.net"); return new URL(sanitizedUrl); } @@ -112,8 +111,8 @@ public class LusciousRipper extends AbstractHTMLRipper { @Override public String normalizeUrl(String url) { try { - return url.toString().replaceFirst( - "^https?:\\/\\/(?:members\\.|legacy\\.)?luscious.net", "https://www.luscious.net"); + return url.replaceFirst( + "^https?://(?:members\\.|legacy\\.)?luscious.net", "https://www.luscious.net"); } catch (Exception e) { LOGGER.info("Error normalizing the url."); LOGGER.error(e); @@ -122,8 +121,8 @@ public class LusciousRipper extends AbstractHTMLRipper { } public class LusciousDownloadThread extends Thread { - private URL url; - private int index; + private final URL url; + private final int index; public LusciousDownloadThread(URL url, int index) { this.url = url; From 64dbb56c62fdcfdb563f5ffab38abe69474d3418 Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 30 Dec 2021 01:57:47 +0100 Subject: [PATCH 250/512] testreport gradle-8 property rename --- build.gradle.kts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 78f6b490..d47626a4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -119,9 +119,9 @@ tasks.withType().configureEach { tasks.jacocoTestReport { dependsOn(tasks.test) // tests are required to run before generating the report reports { - xml.isEnabled = false - csv.isEnabled = false - html.destination = file("${buildDir}/jacocoHtml") + xml.required.set(false) + csv.required.set(false) + html.outputLocation.set(file("${buildDir}/jacocoHtml")) } } From eb9f55664344eca09e4fedb19cc72e4c8d3631db Mon Sep 17 00:00:00 2001 From: neurolancer Date: Sat, 1 Jan 2022 17:34:43 +0000 Subject: [PATCH 251/512] Fix Vsco session token --- .../com/rarchives/ripme/ripper/rippers/VscoRipper.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java index d0a36cdc..e324d5cf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java @@ -11,6 +11,7 @@ import java.util.regex.Pattern; import org.json.JSONObject; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; +import org.jsoup.Connection.Response; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -103,12 +104,12 @@ public class VscoRipper extends AbstractHTMLRipper { String userinfoPage = "https://vsco.co/content/Static/userinfo"; String referer = "https://vsco.co/" + username + "/gallery"; Map cookies = new HashMap<>(); + Map responseCookies = new HashMap<>(); cookies.put("vs_anonymous_id", UUID.randomUUID().toString()); try { - Element doc = Http.url(userinfoPage).cookies(cookies).referrer(referer).ignoreContentType().get().body(); - String json = doc.text().replaceAll("define\\(", ""); - json = json.replaceAll("\\)", ""); - return new JSONObject(json).getString("tkn"); + Response resp = Http.url(userinfoPage).cookies(cookies).referrer(referer).ignoreContentType().response(); + responseCookies = resp.cookies(); + return responseCookies.get("vs"); } catch (IOException e) { LOGGER.error("Could not get user tkn"); return null; From aab6ebd4f7df64d2f9e40e27c397cf510f50b4e7 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 07:55:12 +0100 Subject: [PATCH 252/512] enable vscorippertest --- .../com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java index 44463c76..084d3ce6 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java @@ -29,7 +29,6 @@ public class VscoRipperTest extends RippersTest { * @throws IOException */ @Test - @Tag("flaky") public void testHyphenatedRip() throws IOException { VscoRipper ripper = new VscoRipper(new URL("https://vsco.co/jolly-roger/gallery")); testRipper(ripper); From 822f78bba6a2a6f5207c2498e52afd73dfca199b Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 08:02:58 +0100 Subject: [PATCH 253/512] booru test flaky --- .../com/rarchives/ripme/tst/ripper/rippers/BooruRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BooruRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BooruRipperTest.java index f7918aad..663418b4 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BooruRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BooruRipperTest.java @@ -7,10 +7,12 @@ import java.util.List; import com.rarchives.ripme.ripper.rippers.BooruRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class BooruRipperTest extends RippersTest { @Test + @Tag("flaky") public void testRip() throws IOException { List passURLs = new ArrayList<>(); passURLs.add(new URL("https://xbooru.com/index.php?page=post&s=list&tags=furry")); From af701b21d82b735fdd03f9c561091dd6b4b482e7 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 08:31:29 +0100 Subject: [PATCH 254/512] not build on tags, release already built in future currently 2 builds happen, one for the main branch, one for the tag. this should not be necessary, one build is sufficient, producing a verison including the git hash. this can be converted to a release later on, and ripme.json updated to this build. the release procedure is thus: first commit, and let it build, second, create a tag like '2.0.4' and mark the release as official, then push an updated ripme.json with the version 2.0.4-12-487e38cc to trigger clients prompting to updating ifself when it is run. --- .github/workflows/gradle.yml | 8 +++++++- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 9 ++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index b83caa07..c6aaec51 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -1,6 +1,12 @@ name: CI + release -on: [push, pull_request] +on: + pull_request: + push: + branches: + - '**' + tags: + - '!**' jobs: build: diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index b593d6ec..2fb9246c 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -45,7 +45,14 @@ public class UpdateUtils { } private static String getUpdateJarURL(String latestVersion) { - return "https://github.com/" + REPO_NAME + "/releases/download/" + latestVersion + "/ripme-" + latestVersion + ".jar"; + // this works with a tag created in github, and thus download URLs like: + // https://github.com/ripmeapp2/ripme/releases/download/2.0.4/ripme-2.0.4-12-487e38cc.jar + return "https://github.com/" + + REPO_NAME + + "/releases/download/" + + latestVersion.substring(0, latestVersion.indexOf("-")) + + "/ripme-" + + latestVersion + ".jar"; } public static String getThisJarVersion() { From 9aed081e503f196ba576475660d8be8c510e9d3f Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 08:55:35 +0100 Subject: [PATCH 255/512] print shasum -a 256 after building, only before upload --- .github/workflows/gradle.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index c6aaec51..b081b934 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -50,6 +50,10 @@ jobs: - name: Build with Gradle run: ./gradlew build + - name: SHA256 + if: matrix.upload + run: shasum -a 256 build/libs/*.jar + - name: upload jar as asset if: matrix.upload uses: actions/upload-artifact@v2 From a3054da5db074e1a233f5cefe0fd67fd10cca763 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 09:42:28 +0100 Subject: [PATCH 256/512] danbooru flaky --- .../rarchives/ripme/tst/ripper/rippers/DanbooruRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DanbooruRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DanbooruRipperTest.java index 575864a5..de3c6b5e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DanbooruRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DanbooruRipperTest.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.DanbooruRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -11,6 +12,7 @@ import java.util.List; public class DanbooruRipperTest extends RippersTest { @Test + @Tag("flaky") public void testRip() throws IOException { List passURLs = new ArrayList<>(); passURLs.add(new URL("https://danbooru.donmai.us/posts?tags=brown_necktie")); From 03e32cb71f1a02461f5aeb4029f10d26627b33c1 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 11:02:34 +0100 Subject: [PATCH 257/512] fix download and start java.nio allows to move a file also onto a different filesystem, besides throwing proper error messages. --- .../com/rarchives/ripme/ui/UpdateUtils.java | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 2fb9246c..ac7a4a37 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -20,9 +20,14 @@ import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + public class UpdateUtils { private static final Logger logger = LogManager.getLogger(UpdateUtils.class); @@ -103,8 +108,7 @@ public class UpdateUtils { String latestVersion = ripmeJson.getString("latestVersion"); if (UpdateUtils.isNewerVersion(latestVersion)) { logger.info("Found newer version: " + latestVersion); - logger.info("Downloading new version..."); - logger.info("New version found, downloading..."); + logger.info("Downloading" +getUpdateJarURL(latestVersion) + " ..."); try { UpdateUtils.downloadJarAndLaunch(getUpdateJarURL(latestVersion), false); } catch (IOException e) { @@ -157,7 +161,7 @@ public class UpdateUtils { return; } configUpdateLabel.setText("Downloading new version..."); - logger.info("New version found, downloading..."); + logger.info("New version found, downloading " + getUpdateJarURL(latestVersion)); try { UpdateUtils.downloadJarAndLaunch(getUpdateJarURL(latestVersion), true); } catch (IOException e) { @@ -313,13 +317,13 @@ public class UpdateUtils { // Modifying file and launching it: *nix distributions don't have any issues // with modifying/deleting files // while they are being run - File mainFile = new File(mainFileName); - String mainFilePath = mainFile.getAbsolutePath(); - mainFile.delete(); - new File(updateFileName).renameTo(new File(mainFilePath)); + Path newFile = Paths.get(updateFileName); + Path oldFile = Paths.get(mainFileName); + Files.move(newFile, oldFile, REPLACE_EXISTING); if (shouldLaunch) { // No need to do it during shutdown: the file used will indeed be the new one - Runtime.getRuntime().exec("java -jar " + mainFileName); + logger.info("Executing: " + oldFile); + Runtime.getRuntime().exec("java -jar " + oldFile); } logger.info("Update installed, newer version should be executed upon relaunch"); System.exit(0); From 9911c507b2f548d632dc0c0f8daeefd56955bfa0 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 09:53:01 +0100 Subject: [PATCH 258/512] release 2.0.4, tag 2.0.4-13-03e32cb7 typo in newest version hash --- ripme.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ripme.json b/ripme.json index 3566b498..858c65cb 100644 --- a/ripme.json +++ b/ripme.json @@ -1,7 +1,8 @@ { - "latestVersion": "2.0.3", - "currentHash": "fe2e5ddec836420329a9348ccf5b019087fc750128fe485479e8b7ccee75fc93", + "latestVersion": "2.0.4-13-03e32cb7", + "currentHash": "2d2437911a63f1bc75ed4b761a4cb464bd14f84dea5dab19b122bb35905381b2", "changeList": [ + "2.0.4-13-03e32cb7.: fix vsco, add danbooru.", "2.0.3: Check new version against ripme2app.", "2.0.2: Add greek translation, fixed reddit, redgif.", "2.0.1: Fixed reddit, tujigu, xhamster, imagebam, erome; marked some tests as flaky.", From 1ef49d33e5db0750e24b580d56285adb6053ac1f Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 15:50:47 +0100 Subject: [PATCH 259/512] remove junit-4.13 --- build.gradle.kts | 1 - .../ripme/tst/ripper/rippers/HentainexusRipperTest.java | 5 +++-- .../ripme/tst/ripper/rippers/PorncomixinfoRipperTest.java | 2 +- .../ripme/tst/ripper/rippers/SoundgasmRipperTest.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index d47626a4..398019ba 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -26,7 +26,6 @@ dependencies { implementation("org.graalvm.js:js:21.3.0") testImplementation(enforcedPlatform("org.junit:junit-bom:5.8.2")) testImplementation("org.junit.jupiter:junit-jupiter") - testImplementation("junit:junit:4.13.2") } group = "com.rarchives.ripme" diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java index cad4bb7e..835f40e6 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java @@ -7,11 +7,12 @@ import java.util.List; import com.rarchives.ripme.ripper.rippers.HentaiNexusRipper; import org.json.JSONObject; -import org.junit.Assert; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + public class HentainexusRipperTest extends RippersTest { @Test @Tag("flaky") @@ -41,7 +42,7 @@ public class HentainexusRipperTest extends RippersTest { testOK = false; } - Assert.assertEquals(true, testOK); + assertEquals(true, testOK); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixinfoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixinfoRipperTest.java index 6a839036..10f70ac0 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixinfoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixinfoRipperTest.java @@ -4,7 +4,7 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.PorncomixinfoRipper; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class PorncomixinfoRipperTest extends RippersTest { @Test diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java index 753e7b78..76d7bd20 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java @@ -1,8 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.RedditRipper; -import org.junit.Test; import com.rarchives.ripme.ripper.rippers.SoundgasmRipper; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.net.URL; From 53248d943e76ccac61fc4bc90401f3cb221bd9ff Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 17 Jan 2022 00:13:12 +0100 Subject: [PATCH 260/512] ripme.jar --version prints version on stdout https://github.com/ripmeapp2/ripme/issues/62 --- src/main/java/com/rarchives/ripme/App.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index b65bcbae..c074088c 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -59,7 +59,7 @@ public class App { CommandLine cl = getArgs(args); if (args.length > 0 && cl.hasOption('v')){ - logger.info(UpdateUtils.getThisJarVersion()); + System.out.println(UpdateUtils.getThisJarVersion()); System.exit(0); } From 045084524d31bccc8056f6bc6a23d4005db3fdd8 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 17 Jan 2022 00:43:48 +0100 Subject: [PATCH 261/512] listalripper with _ in url test fixes https://github.com/RipMeApp/ripme/issues/1943 --- .../ripme/ripper/rippers/ListalRipper.java | 2 +- .../ripme/tst/ripper/rippers/ListalRipperTest.java | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java index 8986fd91..fed85531 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java @@ -26,7 +26,7 @@ public class ListalRipper extends AbstractHTMLRipper { private Pattern p1 = Pattern.compile("https:\\/\\/www.listal.com\\/list\\/([a-zA-Z0-9-]+)"); private Pattern p2 = - Pattern.compile("https:\\/\\/www.listal.com\\/((?:(?:[a-zA-Z0-9-]+)\\/?)+)"); + Pattern.compile("https:\\/\\/www.listal.com\\/((?:(?:[a-zA-Z0-9-_%]+)\\/?)+)"); private String listId = null; // listId to get more images via POST. private String postUrl = "https://www.listal.com/item-list/"; //to load more images. private UrlType urlType = UrlType.UNKNOWN; diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java index aba41af3..dc12bbe8 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java @@ -3,6 +3,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ListalRipper; +import org.junit.jupiter.api.Test; public class ListalRipperTest extends RippersTest { @@ -10,6 +11,18 @@ public class ListalRipperTest extends RippersTest { * Test for list type url. * @throws IOException */ + @Test + public void testPictures() throws IOException { + ListalRipper ripper = + new ListalRipper(new URL("https://www.listal.com/emma-stone_iii/pictures")); + testRipper(ripper); + } + + /** + * Test for list type url. + * @throws IOException + */ + @Test public void testRipListType() throws IOException { ListalRipper ripper = new ListalRipper(new URL("https://www.listal.com/list/evolution-emma-stone")); @@ -20,6 +33,7 @@ public class ListalRipperTest extends RippersTest { * Test for folder type url. * @throws IOException */ + @Test public void testRipFolderType() throws IOException { ListalRipper ripper = new ListalRipper(new URL("https://www.listal.com/chet-atkins/pictures")); From 24115fbb66fa16a27ff73c998d953717df68adda Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 12:15:26 +0100 Subject: [PATCH 262/512] updateutils now uses java.nio --- .../com/rarchives/ripme/ui/UpdateUtils.java | 49 ++++++++----------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index ac7a4a37..5ceb361b 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -12,13 +12,10 @@ import org.jsoup.nodes.Document; import javax.swing.*; import java.awt.*; import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; @@ -35,15 +32,15 @@ public class UpdateUtils { private static final String DEFAULT_VERSION = "1.7.94-10-b6345398"; private static final String REPO_NAME = "ripmeapp2/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/main/ripme.json"; - private static final String updateFileName = "ripme.jar.update"; - private static String mainFileName; + private static final Path newFile = Paths.get("ripme.jar.new"); + private static Path mainFile; private static JSONObject ripmeJson; static { try { - mainFileName = new File(UpdateUtils.class.getProtectionDomain().getCodeSource().getLocation().toURI()).getAbsolutePath(); + mainFile = Paths.get(UpdateUtils.class.getProtectionDomain().getCodeSource().getLocation().toURI()); } catch (URISyntaxException | IllegalArgumentException e) { - mainFileName = "ripme.jar"; + mainFile = Paths.get("ripme.jar"); logger.error("Unable to get path of jar"); e.printStackTrace(); } @@ -225,10 +222,10 @@ public class UpdateUtils { } // Code take from https://stackoverflow.com/a/30925550 - public static String createSha256(File file) { + public static String createSha256(Path file) { try { MessageDigest digest = MessageDigest.getInstance("SHA-256"); - InputStream fis = new FileInputStream(file); + InputStream fis = Files.newInputStream(file); int n = 0; byte[] buffer = new byte[8192]; while (n != -1) { @@ -247,7 +244,7 @@ public class UpdateUtils { // lowercase return sb.toString().toLowerCase(); } catch (FileNotFoundException e) { - logger.error("Could not find file: " + file.getName()); + logger.error("Could not find file: " + file); } catch (NoSuchAlgorithmException | IOException e) { logger.error("Got error getting file hash " + e.getMessage()); } @@ -260,13 +257,13 @@ public class UpdateUtils { .timeout(Utils.getConfigInteger("download.timeout", 60 * 1000)).maxBodySize(1024 * 1024 * 100) .execute(); - try (FileOutputStream out = new FileOutputStream(updateFileName)) { + try (OutputStream out = Files.newOutputStream(newFile)) { out.write(response.bodyAsBytes()); } // Only check the hash if the user hasn't disabled hash checking if (Utils.getConfigBoolean("security.check_update_hash", true)) { - String updateHash = createSha256(new File(updateFileName)); - logger.info("Download of new version complete; saved to " + updateFileName); + String updateHash = createSha256(newFile); + logger.info("Download of new version complete; saved to " + newFile); logger.info("Checking hash of update"); if (!ripmeJson.getString("currentHash").equals(updateHash)) { @@ -281,19 +278,17 @@ public class UpdateUtils { if (System.getProperty("os.name").toLowerCase().contains("win")) { // Windows - final String batchFile = "update_ripme.bat"; - final String batchPath = new File(batchFile).getAbsolutePath(); + final Path batchFile = Paths.get("update_ripme.bat"); String script = "@echo off\r\n" + "timeout 1\r\n" - + "copy \"" + updateFileName + "\" \"" + mainFileName + "\"\r\n" - + "del \"" + updateFileName + "\"\r\n"; + + "copy \"" + newFile + "\" \"" + mainFile + "\"\r\n" + + "del \"" + newFile + "\"\r\n"; if (shouldLaunch) - script += "\"" + mainFileName + "\"\r\n"; - script += "del \"" + batchPath + "\"\r\n"; + script += "\"" + mainFile + "\"\r\n"; + script += "del \"" + batchFile + "\"\r\n"; - final String[] batchExec = new String[]{batchPath}; // Create updater script - try (BufferedWriter bw = new BufferedWriter(new FileWriter(batchFile))) { + try (BufferedWriter bw = Files.newBufferedWriter(batchFile)) { bw.write(script); bw.flush(); } @@ -303,7 +298,7 @@ public class UpdateUtils { Runtime.getRuntime().addShutdownHook(new Thread(() -> { try { logger.info("Executing: " + batchFile); - Runtime.getRuntime().exec(batchExec); + Runtime.getRuntime().exec(String.valueOf(batchFile)); } catch (IOException e) { // TODO implement proper stack trace handling this is really just intented as a // placeholder until you implement proper error handling @@ -317,13 +312,11 @@ public class UpdateUtils { // Modifying file and launching it: *nix distributions don't have any issues // with modifying/deleting files // while they are being run - Path newFile = Paths.get(updateFileName); - Path oldFile = Paths.get(mainFileName); - Files.move(newFile, oldFile, REPLACE_EXISTING); + Files.move(newFile, mainFile, REPLACE_EXISTING); if (shouldLaunch) { // No need to do it during shutdown: the file used will indeed be the new one - logger.info("Executing: " + oldFile); - Runtime.getRuntime().exec("java -jar " + oldFile); + logger.info("Executing: " + mainFile); + Runtime.getRuntime().exec("java -jar " + mainFile); } logger.info("Update installed, newer version should be executed upon relaunch"); System.exit(0); From 74291358d28d6ca3f52b1ea038dafa815e55f671 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 13:27:44 +0100 Subject: [PATCH 263/512] app convert to java.nio --- src/main/java/com/rarchives/ripme/App.java | 23 +++++++++++----------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index c074088c..c933dec4 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -1,11 +1,10 @@ package com.rarchives.ripme; import java.awt.*; -import java.io.File; -import java.io.IOException; import java.io.BufferedReader; -import java.io.FileReader; +import java.io.File; import java.io.FileNotFoundException; +import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -254,9 +253,9 @@ public class App { //Read URLs from File if (cl.hasOption('f')) { - String filename = cl.getOptionValue('f'); + Path urlfile = Paths.get(cl.getOptionValue('f')); - try (BufferedReader br = new BufferedReader(new FileReader(filename))) { + try (BufferedReader br = Files.newBufferedReader(urlfile)) { String url; while ((url = br.readLine()) != null) { if (url.startsWith("//") || url.startsWith("#")) { @@ -350,19 +349,18 @@ public class App { /** * Loads history from history file into memory. - * @see MainWindow.loadHistory */ private static void loadHistory() { - File historyFile = new File(Utils.getConfigDir() + File.separator + "history.json"); + Path historyFile = Paths.get(Utils.getConfigDir() + "/history.json"); HISTORY.clear(); - if (historyFile.exists()) { + if (Files.exists(historyFile)) { try { - logger.info("Loading history from " + historyFile.getCanonicalPath()); - HISTORY.fromFile(historyFile.getCanonicalPath()); + logger.info("Loading history from " + historyFile); + HISTORY.fromFile(historyFile.toString()); } catch (IOException e) { logger.error("Failed to load history from file " + historyFile, e); logger.warn( - "RipMe failed to load the history file at " + historyFile.getAbsolutePath() + "\n\n" + + "RipMe failed to load the history file at " + historyFile + "\n\n" + "Error: " + e.getMessage() + "\n\n" + "Closing RipMe will automatically overwrite the contents of this file,\n" + "so you may want to back the file up before closing RipMe!"); @@ -374,6 +372,7 @@ public class App { // Loaded from config, still no entries. // Guess rip history based on rip folder String[] dirs = Utils.getWorkingDirectory().list((dir, file) -> new File(dir.getAbsolutePath() + File.separator + file).isDirectory()); + assert dirs != null; for (String dir : dirs) { String url = RipUtils.urlFromDirectoryName(dir); if (url != null) { @@ -391,7 +390,7 @@ public class App { * @see MainWindow.saveHistory */ private static void saveHistory() { - Path historyFile = Paths.get(Utils.getConfigDir() + File.separator + "history.json"); + Path historyFile = Paths.get(Utils.getConfigDir() + "/history.json"); try { if (!Files.exists(historyFile)) { Files.createDirectories(historyFile.getParent()); From a610d5c3cea5022177b8f9a65c89ff247829ecef Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 13:40:05 +0100 Subject: [PATCH 264/512] utils, java.nio for config file load --- src/main/java/com/rarchives/ripme/utils/Utils.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index e4337e8f..b6d2b973 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -72,9 +72,9 @@ public class Utils { try { String configPath = getConfigFilePath(); - File file = new File(configPath); + Path file = Paths.get(configPath); - if (!file.exists()) { + if (!Files.exists(file)) { // Use default bundled with .jar configPath = CONFIG_FILE; } @@ -82,7 +82,7 @@ public class Utils { config = new PropertiesConfiguration(configPath); LOGGER.info("Loaded " + config.getPath()); - if (file.exists()) { + if (Files.exists(file)) { // Config was loaded from file if (!config.containsKey("twitter.auth") || !config.containsKey("twitter.max_requests") || !config.containsKey("tumblr.auth") || !config.containsKey("error.skip404") @@ -92,7 +92,7 @@ public class Utils { // Need to reload the default config // See https://github.com/4pr0n/ripme/issues/158 LOGGER.warn("Config does not contain key fields, deleting old config"); - file.delete(); + Files.delete(file); config = new PropertiesConfiguration(CONFIG_FILE); LOGGER.info("Loaded " + config.getPath()); } From fca100cd25b73a380ed48899cf552cf76c305641 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 13:57:00 +0100 Subject: [PATCH 265/512] utils, clearurl now java.nio --- src/main/java/com/rarchives/ripme/utils/Utils.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index b6d2b973..9498af52 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -297,8 +297,12 @@ public class Utils { * Delete the url history file */ public static void clearURLHistory() { - File file = new File(getURLHistoryFile()); - file.delete(); + Path file = Paths.get(getURLHistoryFile()); + try { + Files.delete(file); + } catch (IOException e) { + e.printStackTrace(); + } } /** From ffa11e8aa7b79a42390ed6132be4a700c70525c1 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 14:02:19 +0100 Subject: [PATCH 266/512] shortenSaveAsWindows now uses java.nio --- .../com/rarchives/ripme/ripper/DownloadFileThread.java | 3 ++- src/main/java/com/rarchives/ripme/utils/Utils.java | 4 ++-- src/test/java/com/rarchives/ripme/tst/UtilsTest.java | 7 ++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 9b515d1d..9b99f747 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -5,6 +5,7 @@ import java.net.HttpURLConnection; import java.net.SocketTimeoutException; import java.net.URL; import java.net.URLConnection; +import java.nio.file.Files; import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; @@ -237,7 +238,7 @@ class DownloadFileThread extends Thread { } else if (saveAs.getAbsolutePath().length() > 259 && Utils.isWindows()) { // This if is for when the file path has gone above 260 chars which windows does // not allow - fos = new FileOutputStream( + fos = Files.newOutputStream( Utils.shortenSaveAsWindows(saveAs.getParentFile().getPath(), saveAs.getName())); } } diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 9498af52..9e757a09 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -891,7 +891,7 @@ public class Utils { return fileNameToSan.replaceAll("[\\\\/:*?\"<>|]", "_"); } - public static File shortenSaveAsWindows(String ripsDirPath, String fileName) throws FileNotFoundException { + public static Path shortenSaveAsWindows(String ripsDirPath, String fileName) throws FileNotFoundException { LOGGER.error("The filename " + fileName + " is to long to be saved on this file system."); LOGGER.info("Shortening filename"); String fullPath = ripsDirPath + File.separator + fileName; @@ -909,7 +909,7 @@ public class Utils { fullPath = fullPath.substring(0, 259 - pathLength - fileExt.length() + 1) + "." + fileExt; LOGGER.info(fullPath); LOGGER.info(fullPath.length()); - return new File(fullPath); + return Paths.get(fullPath); } } diff --git a/src/test/java/com/rarchives/ripme/tst/UtilsTest.java b/src/test/java/com/rarchives/ripme/tst/UtilsTest.java index 3a8a8668..c43fa76a 100644 --- a/src/test/java/com/rarchives/ripme/tst/UtilsTest.java +++ b/src/test/java/com/rarchives/ripme/tst/UtilsTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst; -import java.io.File; import java.io.FileNotFoundException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import com.rarchives.ripme.utils.Utils; @@ -60,8 +61,8 @@ public class UtilsTest { public void testShortenFileNameWindows() throws FileNotFoundException { String filename = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff.png"; // Test filename shortening for windows - File f = Utils.shortenSaveAsWindows("D:/rips/test/reddit/deep", filename); - Assertions.assertEquals(new File( + Path f = Utils.shortenSaveAsWindows("D:/rips/test/reddit/deep", filename); + Assertions.assertEquals(Paths.get( "D:/rips/test/reddit/deep/ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff.png"), f); } From 6bbd75a824c5e752c3a930d38a6cc4856f074e47 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 14:13:16 +0100 Subject: [PATCH 267/512] remove unused fuzzyexists --- .../ripme/ripper/DownloadFileThread.java | 2 +- .../java/com/rarchives/ripme/utils/Utils.java | 23 +------------------ 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 9b99f747..f879e069 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -81,7 +81,7 @@ class DownloadFileThread extends Thread { return; } if (saveAs.exists() && !observer.tryResumeDownload() && !getFileExtFromMIME - || Utils.fuzzyExistsBetter(Paths.get(saveAs.getParent()), saveAs.getName()) && getFileExtFromMIME + || Utils.fuzzyExists(Paths.get(saveAs.getParent()), saveAs.getName()) && getFileExtFromMIME && !observer.tryResumeDownload()) { if (Utils.getConfigBoolean("file.overwrite", false)) { logger.info("[!] " + Utils.getLocalizedString("deleting.existing.file") + prettySaveAs); diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 9e757a09..7352c4ac 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -862,28 +862,7 @@ public class Utils { } // Checks if a file exists ignoring it's extension. - // Code from: https://stackoverflow.com/a/17698068 - public static boolean fuzzyExists(File folder, String fileName) { - if (!folder.exists()) { - return false; - } - File[] listOfFiles = folder.listFiles(); - if (listOfFiles == null) { - return false; - } - - for (File file : listOfFiles) { - if (file.isFile()) { - String[] filename = file.getName().split("\\.(?=[^.]+$)"); // split filename from it's extension - if (filename[0].equalsIgnoreCase(fileName)) { - return true; - } - } - } - return false; - } - - public static boolean fuzzyExistsBetter(Path folder, String filename) { + public static boolean fuzzyExists(Path folder, String filename) { return Files.exists(folder.resolve(filename)); } From cd85219ac875fa7a709c96f6ce126d09827184f9 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 15:24:43 +0100 Subject: [PATCH 268/512] workingdir now java.nio --- src/main/java/com/rarchives/ripme/App.java | 62 ++++---- .../ripme/ripper/AbstractHTMLRipper.java | 5 +- .../ripme/ripper/AbstractJSONRipper.java | 5 +- .../rarchives/ripme/ripper/AlbumRipper.java | 5 +- .../rarchives/ripme/ripper/VideoRipper.java | 5 +- .../com/rarchives/ripme/ui/MainWindow.java | 138 +++++++++--------- .../java/com/rarchives/ripme/utils/Utils.java | 39 ++--- 7 files changed, 128 insertions(+), 131 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index c933dec4..5456a312 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -1,29 +1,5 @@ package com.rarchives.ripme; -import java.awt.*; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; - -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; - -import javax.swing.SwingUtilities; - -import org.apache.commons.cli.BasicParser; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.lang.SystemUtils; - import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.ui.History; import com.rarchives.ripme.ui.HistoryEntry; @@ -32,9 +8,30 @@ import com.rarchives.ripme.ui.UpdateUtils; import com.rarchives.ripme.utils.Proxy; import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; +import org.apache.commons.cli.BasicParser; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.lang.SystemUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import javax.swing.*; +import java.awt.*; +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.stream.Stream; + /** * Entry point to application. * This is where all the fun happens, with the main method. @@ -54,7 +51,7 @@ public class App { * * @param args Array of command line arguments. */ - public static void main(String[] args) { + public static void main(String[] args) throws IOException { CommandLine cl = getArgs(args); if (args.length > 0 && cl.hasOption('v')){ @@ -122,7 +119,7 @@ public class App { * For dealing with command-line arguments. * @param args Array of Command-line arguments */ - private static void handleArguments(String[] args) { + private static void handleArguments(String[] args) throws IOException { CommandLine cl = getArgs(args); //Help (list commands) @@ -350,7 +347,7 @@ public class App { /** * Loads history from history file into memory. */ - private static void loadHistory() { + private static void loadHistory() throws IOException { Path historyFile = Paths.get(Utils.getConfigDir() + "/history.json"); HISTORY.clear(); if (Files.exists(historyFile)) { @@ -371,17 +368,18 @@ public class App { if (HISTORY.toList().isEmpty()) { // Loaded from config, still no entries. // Guess rip history based on rip folder - String[] dirs = Utils.getWorkingDirectory().list((dir, file) -> new File(dir.getAbsolutePath() + File.separator + file).isDirectory()); - assert dirs != null; - for (String dir : dirs) { - String url = RipUtils.urlFromDirectoryName(dir); + Stream stream = Files.list(Utils.getWorkingDirectory()) + .filter(Files::isDirectory); + + stream.forEach(dir -> { + String url = RipUtils.urlFromDirectoryName(dir.toString()); if (url != null) { // We found one, add it to history HistoryEntry entry = new HistoryEntry(); entry.url = url; HISTORY.add(entry); } - } + }); } } } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 30784f59..5afaf936 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -6,6 +6,7 @@ import java.io.FileWriter; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -436,7 +437,9 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { */ @Override public void setWorkingDir(URL url) throws IOException { - String path = Utils.getWorkingDirectory().getCanonicalPath(); + Path wd = Utils.getWorkingDirectory(); + // TODO - change to nio + String path = wd.toAbsolutePath().toString(); if (!path.endsWith(File.separator)) { path += File.separator; } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index b63c44c3..6be88472 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -10,6 +10,7 @@ import java.io.FileWriter; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.Path; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -275,7 +276,9 @@ public abstract class AbstractJSONRipper extends AbstractRipper { */ @Override public void setWorkingDir(URL url) throws IOException { - String path = Utils.getWorkingDirectory().getCanonicalPath(); + Path wd = Utils.getWorkingDirectory(); + // TODO - change to nio + String path = wd.toAbsolutePath().toString(); if (!path.endsWith(File.separator)) { path += File.separator; } diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index f245ba62..554c0e54 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -5,6 +5,7 @@ import java.io.FileWriter; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.Path; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -184,7 +185,9 @@ public abstract class AlbumRipper extends AbstractRipper { */ @Override public void setWorkingDir(URL url) throws IOException { - String path = Utils.getWorkingDirectory().getCanonicalPath(); + Path wd = Utils.getWorkingDirectory(); + // TODO - change to nio + String path = wd.toAbsolutePath().toString(); if (!path.endsWith(File.separator)) { path += File.separator; } diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 4fb0f32a..9e935d62 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -9,6 +9,7 @@ import java.io.FileWriter; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.Path; import java.util.Map; @@ -83,7 +84,9 @@ public abstract class VideoRipper extends AbstractRipper { */ @Override public void setWorkingDir(URL url) throws IOException { - String path = Utils.getWorkingDirectory().getCanonicalPath(); + Path wd = Utils.getWorkingDirectory(); + // TODO - change to nio + String path = wd.toAbsolutePath().toString(); if (!path.endsWith(File.separator)) { path += File.separator; diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index dbe16245..430c5460 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1,54 +1,5 @@ package com.rarchives.ripme.ui; -import java.awt.*; -import java.awt.TrayIcon.MessageType; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; -import java.awt.event.MouseAdapter; -import java.awt.event.MouseEvent; -import java.awt.event.WindowAdapter; -import java.awt.event.WindowEvent; -import java.io.*; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; -import java.util.List; - -import javax.imageio.ImageIO; -import javax.swing.DefaultListModel; -import javax.swing.ImageIcon; -import javax.swing.JButton; -import javax.swing.JCheckBox; -import javax.swing.JComboBox; -import javax.swing.JFileChooser; -import javax.swing.JFrame; -import javax.swing.JLabel; -import javax.swing.JList; -import javax.swing.JOptionPane; -import javax.swing.JPanel; -import javax.swing.JProgressBar; -import javax.swing.JScrollPane; -import javax.swing.JTable; -import javax.swing.JTextField; -import javax.swing.JTextPane; -import javax.swing.ListSelectionModel; -import javax.swing.SwingUtilities; -import javax.swing.UIManager; -import javax.swing.border.EmptyBorder; -import javax.swing.event.DocumentEvent; -import javax.swing.event.DocumentListener; -import javax.swing.event.ListDataEvent; -import javax.swing.event.ListDataListener; -import javax.swing.table.AbstractTableModel; -import javax.swing.text.BadLocationException; -import javax.swing.text.SimpleAttributeSet; -import javax.swing.text.StyleConstants; -import javax.swing.text.StyledDocument; - import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; @@ -59,7 +10,42 @@ import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.config.Configuration; import org.apache.logging.log4j.core.config.LoggerConfig; -import javax.swing.UnsupportedLookAndFeelException; +import javax.imageio.ImageIO; +import javax.swing.*; +import javax.swing.border.EmptyBorder; +import javax.swing.event.DocumentEvent; +import javax.swing.event.DocumentListener; +import javax.swing.event.ListDataEvent; +import javax.swing.event.ListDataListener; +import javax.swing.table.AbstractTableModel; +import javax.swing.text.BadLocationException; +import javax.swing.text.SimpleAttributeSet; +import javax.swing.text.StyleConstants; +import javax.swing.text.StyledDocument; +import java.awt.*; +import java.awt.TrayIcon.MessageType; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.awt.event.MouseAdapter; +import java.awt.event.MouseEvent; +import java.awt.event.WindowAdapter; +import java.awt.event.WindowEvent; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.Date; +import java.util.Enumeration; +import java.util.List; +import java.util.stream.Stream; /** * Everything UI-related starts and ends here. @@ -176,7 +162,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { queueListModel.addElement(url); } - public MainWindow() { + public MainWindow() throws IOException { mainFrame = new JFrame("RipMe v" + UpdateUtils.getThisJarVersion()); mainFrame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); mainFrame.setLayout(new GridBagLayout()); @@ -548,7 +534,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { setLogLevel(configLogLevelCombobox.getSelectedItem().toString()); configSaveDirLabel = new JLabel(); try { - String workingDir = (Utils.shortenPath(Utils.getWorkingDirectory())); + String workingDir = (Utils.shortenPath(Utils.getWorkingDirectory().toString())); configSaveDirLabel.setText(workingDir); configSaveDirLabel.setForeground(Color.BLUE); configSaveDirLabel.setCursor(new Cursor(Cursor.HAND_CURSOR)); @@ -883,17 +869,23 @@ public final class MainWindow implements Runnable, RipStatusHandler { configSaveDirLabel.addMouseListener(new MouseAdapter() { @Override public void mouseClicked(MouseEvent e) { - File file = new File(Utils.getWorkingDirectory().toString()); - Desktop desktop = Desktop.getDesktop(); + Path file = null; try { - desktop.open(file); - } catch (Exception e1) { + file = Utils.getWorkingDirectory(); + Desktop desktop = Desktop.getDesktop(); + desktop.open(file.toFile()); + } catch (IOException ex) { } } }); configSaveDirButton.addActionListener(arg0 -> { UIManager.put("FileChooser.useSystemExtensionHiding", false); - JFileChooser jfc = new JFileChooser(Utils.getWorkingDirectory()); + JFileChooser jfc = null; + try { + jfc = new JFileChooser(Utils.getWorkingDirectory().toString()); + } catch (IOException e) { + e.printStackTrace(); + } jfc.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); int returnVal = jfc.showDialog(null, "select directory"); if (returnVal != JFileChooser.APPROVE_OPTION) { @@ -912,7 +904,12 @@ public final class MainWindow implements Runnable, RipStatusHandler { }); configUrlFileChooserButton.addActionListener(arg0 -> { UIManager.put("FileChooser.useSystemExtensionHiding", false); - JFileChooser jfc = new JFileChooser(Utils.getWorkingDirectory()); + JFileChooser jfc = null; + try { + jfc = new JFileChooser(Utils.getWorkingDirectory().toAbsolutePath().toString()); + } catch (IOException e) { + e.printStackTrace(); + } jfc.setFileSelectionMode(JFileChooser.FILES_ONLY); int returnVal = jfc.showDialog(null, "Open"); if (returnVal != JFileChooser.APPROVE_OPTION) { @@ -1157,7 +1154,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { LOGGER.error(line); } - private void loadHistory() { + private void loadHistory() throws IOException { File historyFile = new File(Utils.getConfigDir() + File.separator + "history.json"); HISTORY.clear(); if (historyFile.exists()) { @@ -1177,19 +1174,18 @@ public final class MainWindow implements Runnable, RipStatusHandler { if (HISTORY.toList().isEmpty()) { // Loaded from config, still no entries. // Guess rip history based on rip folder - String[] dirs = Utils.getWorkingDirectory() - .list((dir, file) -> new File(dir.getAbsolutePath() + File.separator + file).isDirectory()); - if (dirs != null) { - for (String dir : dirs) { - String url = RipUtils.urlFromDirectoryName(dir); - if (url != null) { - // We found one, add it to history - HistoryEntry entry = new HistoryEntry(); - entry.url = url; - HISTORY.add(entry); - } + Stream stream = Files.list(Utils.getWorkingDirectory()) + .filter(Files::isDirectory); + + stream.forEach(dir -> { + String url = RipUtils.urlFromDirectoryName(dir.toString()); + if (url != null) { + // We found one, add it to history + HistoryEntry entry = new HistoryEntry(); + entry.url = url; + HISTORY.add(entry); } - } + }); } } } diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 7352c4ac..6edcc71e 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -109,21 +109,16 @@ public class Utils { * * @return Root directory to save rips to. */ - public static File getWorkingDirectory() { - String currentDir = ""; - try { - currentDir = getJarDirectory().getCanonicalPath() + File.separator + RIP_DIRECTORY + File.separator; - } catch (IOException e) { - LOGGER.error("Error while finding working dir: ", e); - } + public static Path getWorkingDirectory() throws IOException { + String currentDir = getJarDirectory() + File.separator + RIP_DIRECTORY + File.separator; if (config != null) { currentDir = getConfigString("rips.directory", currentDir); } - File workingDir = new File(currentDir); - if (!workingDir.exists()) { - workingDir.mkdirs(); + Path workingDir = Paths.get(currentDir); + if (!Files.exists(workingDir)) { + Files.createDirectory(workingDir); } return workingDir; } @@ -240,13 +235,13 @@ public class Utils { + File.separator + "ripme"; } - private static File getJarDirectory() { - File jarDirectory = Objects.requireNonNull(Utils.class.getResource("/rip.properties")).toString().contains("jar:") - ? new File(System.getProperty("java.class.path")).getParentFile() - : new File(System.getProperty("user.dir")); + private static Path getJarDirectory() { + Path jarDirectory = Objects.requireNonNull(Utils.class.getResource("/rip.properties")).toString().contains("jar:") + ? Paths.get(System.getProperty("java.class.path")).getParent() + : Paths.get(System.getProperty("user.dir")); if (jarDirectory == null) - jarDirectory = new File("."); + jarDirectory = Paths.get("."); return jarDirectory; } @@ -255,13 +250,9 @@ public class Utils { * Determines if the app is running in a portable mode. i.e. on a USB stick */ private static boolean portableMode() { - try { - File file = new File(getJarDirectory().getCanonicalPath() + File.separator + CONFIG_FILE); - if (file.exists() && !file.isDirectory()) { - return true; - } - } catch (IOException e) { - return false; + Path file = getJarDirectory().resolve(CONFIG_FILE); + if (Files.exists(file) && !Files.isDirectory(file)) { + return true; } return false; @@ -273,7 +264,7 @@ public class Utils { public static String getConfigDir() { if (portableMode()) { try { - return getJarDirectory().getCanonicalPath(); + return getJarDirectory().toAbsolutePath().toString(); } catch (Exception e) { return "."; } @@ -287,7 +278,7 @@ public class Utils { return getUnixConfigDir(); try { - return getJarDirectory().getCanonicalPath(); + return getJarDirectory().toAbsolutePath().toString(); } catch (Exception e) { return "."; } From a58171477900c933b710443dbbaa74ab72902e8e Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 16:08:12 +0100 Subject: [PATCH 269/512] reformat mangadex --- .../ripme/ripper/rippers/MangadexRipper.java | 67 +++++++++---------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java index cfe2e53f..d9a80080 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java @@ -1,39 +1,40 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractJSONRipper; -import com.rarchives.ripme.ui.History; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.utils.Http; -import com.rarchives.ripme.utils.Utils; import org.json.JSONArray; import org.json.JSONObject; -import org.jsoup.Connection; -import org.jsoup.nodes.Document; -import java.io.File; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; public class MangadexRipper extends AbstractJSONRipper { - private String chapterApiEndPoint = "https://mangadex.org/api/chapter/"; - private String mangaApiEndPoint = "https://mangadex.org/api/manga/"; + private final String chapterApiEndPoint = "https://mangadex.org/api/chapter/"; + private final String mangaApiEndPoint = "https://mangadex.org/api/manga/"; private boolean isSingleChapter; - private String getImageUrl(String chapterHash, String imageName, String server) { - return server + chapterHash + "/" + imageName; - } public MangadexRipper(URL url) throws IOException { super(url); } + private String getImageUrl(String chapterHash, String imageName, String server) { + return server + chapterHash + "/" + imageName; + } + @Override public String getHost() { return "mangadex"; } + @Override public String getDomain() { return "mangadex.org"; @@ -49,14 +50,12 @@ public class MangadexRipper extends AbstractJSONRipper { String capID = getChapterID(url.toExternalForm()); String mangaID = getMangaID(url.toExternalForm()); if (capID != null) { - isSingleChapter=true; + isSingleChapter = true; return capID; + } else if (mangaID != null) { + isSingleChapter = false; + return mangaID; } - else - if(mangaID!=null){ - isSingleChapter=false; - return mangaID; - } throw new MalformedURLException("Unable to get chapter ID from" + url); } @@ -68,10 +67,11 @@ public class MangadexRipper extends AbstractJSONRipper { } return null; } - private String getMangaID(String url){ + + private String getMangaID(String url) { Pattern p = Pattern.compile("https://mangadex.org/title/([\\d]+)/(.+)"); Matcher m = p.matcher(url); - if(m.matches()){ + if (m.matches()) { return m.group(1); } return null; @@ -83,16 +83,15 @@ public class MangadexRipper extends AbstractJSONRipper { // Get the chapter ID String chapterID = getChapterID(url.toExternalForm()); String mangaID = getMangaID(url.toExternalForm()); - if(mangaID!=null){ - return Http.url(new URL(mangaApiEndPoint+mangaID)).getJSON(); - } - else + if (mangaID != null) { + return Http.url(new URL(mangaApiEndPoint + mangaID)).getJSON(); + } else return Http.url(new URL(chapterApiEndPoint + chapterID)).getJSON(); } @Override protected List getURLsFromJSON(JSONObject json) { - if(isSingleChapter){ + if (isSingleChapter) { List assetURLs = new ArrayList<>(); JSONArray currentObject; String chapterHash; @@ -110,12 +109,12 @@ public class MangadexRipper extends AbstractJSONRipper { JSONObject chaptersJSON = (JSONObject) json.get("chapter"); JSONObject temp; Iterator keys = chaptersJSON.keys(); - HashMap chapterIDs = new HashMap<>(); + HashMap chapterIDs = new HashMap<>(); while (keys.hasNext()) { - String keyValue = (String) keys.next(); - temp=(JSONObject)chaptersJSON.get(keyValue); - if(temp.getString("lang_name").equals("English")) { - chapterIDs.put(temp.getDouble("chapter"),keyValue); + String keyValue = keys.next(); + temp = (JSONObject) chaptersJSON.get(keyValue); + if (temp.getString("lang_name").equals("English")) { + chapterIDs.put(temp.getDouble("chapter"), keyValue); } } @@ -125,17 +124,17 @@ public class MangadexRipper extends AbstractJSONRipper { String chapterHash; // Server is the cdn hosting the images. String server; - JSONObject chapterJSON=null; - TreeMap treeMap = new TreeMap<>(chapterIDs); + JSONObject chapterJSON = null; + TreeMap treeMap = new TreeMap<>(chapterIDs); Iterator it = treeMap.keySet().iterator(); - while(it.hasNext()) { - double key =(double) it.next(); + while (it.hasNext()) { + double key = (double) it.next(); try { chapterJSON = Http.url(new URL(chapterApiEndPoint + treeMap.get(key))).getJSON(); } catch (IOException e) { e.printStackTrace(); } - sendUpdate(RipStatusMessage.STATUS.LOADING_RESOURCE,"chapter "+key); + sendUpdate(RipStatusMessage.STATUS.LOADING_RESOURCE, "chapter " + key); chapterHash = chapterJSON.getString("hash"); server = chapterJSON.getString("server"); for (int i = 0; i < chapterJSON.getJSONArray("page_array").length(); i++) { From 09d21cd134016797d30380baa65a46e18b93e3df Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 16:10:50 +0100 Subject: [PATCH 270/512] reformat ehentai --- .../ripme/ripper/rippers/EHentaiRipper.java | 78 +++++++++---------- 1 file changed, 37 insertions(+), 41 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java index 3cdbae4e..76ddf7ed 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java @@ -1,5 +1,16 @@ package com.rarchives.ripme.ripper.rippers; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.ripper.DownloadThreadPool; +import com.rarchives.ripme.ui.RipStatusMessage; +import com.rarchives.ripme.ui.RipStatusMessage.STATUS; +import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.RipUtils; +import com.rarchives.ripme.utils.Utils; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + import java.io.File; import java.io.IOException; import java.net.MalformedURLException; @@ -11,46 +22,33 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.rarchives.ripme.ui.RipStatusMessage; -import com.rarchives.ripme.utils.RipUtils; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; -import com.rarchives.ripme.utils.Http; -import com.rarchives.ripme.utils.Utils; - public class EHentaiRipper extends AbstractHTMLRipper { // All sleep times are in milliseconds - private static final int PAGE_SLEEP_TIME = 3000; - private static final int IMAGE_SLEEP_TIME = 1500; - private static final int IP_BLOCK_SLEEP_TIME = 60 * 1000; + private static final int PAGE_SLEEP_TIME = 3000; + private static final int IMAGE_SLEEP_TIME = 1500; + private static final int IP_BLOCK_SLEEP_TIME = 60 * 1000; + private static final Map cookies = new HashMap<>(); - private String lastURL = null; - - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool ehentaiThreadPool = new DownloadThreadPool("ehentai"); - @Override - public DownloadThreadPool getThreadPool() { - return ehentaiThreadPool; - } - - // Current HTML document - private Document albumDoc = null; - - private static final Map cookies = new HashMap<>(); static { cookies.put("nw", "1"); cookies.put("tip", "1"); } + private String lastURL = null; + // Thread pool for finding direct image links from "image" pages (html) + private final DownloadThreadPool ehentaiThreadPool = new DownloadThreadPool("ehentai"); + // Current HTML document + private Document albumDoc = null; + public EHentaiRipper(URL url) throws IOException { super(url); } + @Override + public DownloadThreadPool getThreadPool() { + return ehentaiThreadPool; + } + @Override public String getHost() { return "e-hentai"; @@ -95,6 +93,7 @@ public class EHentaiRipper extends AbstractHTMLRipper { /** * Attempts to get page, checks for IP ban, waits. + * * @param url * @return Page document * @throws IOException If page loading errors, or if retries are exhausted @@ -106,9 +105,9 @@ public class EHentaiRipper extends AbstractHTMLRipper { sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm()); LOGGER.info("Retrieving " + url); doc = Http.url(url) - .referrer(this.url) - .cookies(cookies) - .get(); + .referrer(this.url) + .cookies(cookies) + .get(); if (doc.toString().contains("IP address will be automatically banned")) { if (retries == 0) { throw new IOException("Hit rate limit and maximum number of retries, giving up"); @@ -120,8 +119,7 @@ public class EHentaiRipper extends AbstractHTMLRipper { } catch (InterruptedException e) { throw new IOException("Interrupted while waiting for rate limit to subside"); } - } - else { + } else { return doc; } } @@ -197,21 +195,20 @@ public class EHentaiRipper extends AbstractHTMLRipper { ehentaiThreadPool.addThread(t); try { Thread.sleep(IMAGE_SLEEP_TIME); - } - catch (InterruptedException e) { + } catch (InterruptedException e) { LOGGER.warn("Interrupted while waiting to load next image", e); } } /** * Helper class to find and download images found on "image" pages - * + *

* Handles case when site has IP-banned the user. */ private class EHentaiImageThread extends Thread { - private URL url; - private int index; - private File workingDir; + private final URL url; + private final int index; + private final File workingDir; EHentaiImageThread(URL url, int index, File workingDir) { super(); @@ -252,8 +249,7 @@ public class EHentaiRipper extends AbstractHTMLRipper { } savePath += m.group(1); addURLToDownload(new URL(imgsrc), new File(savePath)); - } - else { + } else { // Provide prefix and let the AbstractRipper "guess" the filename String prefix = ""; if (Utils.getConfigBoolean("download.save_order", true)) { From 627152853ea65389f3f790cba6bf40ed649b969b Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 16:26:37 +0100 Subject: [PATCH 271/512] ehentai java.nio --- .../ripme/ripper/rippers/EHentaiRipper.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java index 76ddf7ed..13568758 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java @@ -11,10 +11,11 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; -import java.io.File; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -191,7 +192,7 @@ public class EHentaiRipper extends AbstractHTMLRipper { @Override public void downloadURL(URL url, int index) { - EHentaiImageThread t = new EHentaiImageThread(url, index, this.workingDir); + EHentaiImageThread t = new EHentaiImageThread(url, index, this.workingDir.toPath()); ehentaiThreadPool.addThread(t); try { Thread.sleep(IMAGE_SLEEP_TIME); @@ -208,9 +209,9 @@ public class EHentaiRipper extends AbstractHTMLRipper { private class EHentaiImageThread extends Thread { private final URL url; private final int index; - private final File workingDir; + private final Path workingDir; - EHentaiImageThread(URL url, int index, File workingDir) { + EHentaiImageThread(URL url, int index, Path workingDir) { super(); this.url = url; this.index = index; @@ -243,12 +244,12 @@ public class EHentaiRipper extends AbstractHTMLRipper { Matcher m = p.matcher(imgsrc); if (m.matches()) { // Manually discover filename from URL - String savePath = this.workingDir + File.separator; + String savePath = this.workingDir + "/"; if (Utils.getConfigBoolean("download.save_order", true)) { savePath += String.format("%03d_", index); } savePath += m.group(1); - addURLToDownload(new URL(imgsrc), new File(savePath)); + addURLToDownload(new URL(imgsrc), Paths.get(savePath).toFile()); } else { // Provide prefix and let the AbstractRipper "guess" the filename String prefix = ""; From e3f6499840a551848d5ec81825ba7b6597a62dd3 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 16:29:19 +0100 Subject: [PATCH 272/512] multiple rippers java.nio --- .../ripper/rippers/FivehundredpxRipper.java | 9 ++-- .../ripper/rippers/FuraffinityRipper.java | 36 ++++++++------- .../ripme/ripper/rippers/PahealRipper.java | 13 +++--- .../ripme/ripper/rippers/PornhubRipper.java | 6 +-- .../ripme/ripper/rippers/RedditRipper.java | 44 +++++++++++-------- 5 files changed, 59 insertions(+), 49 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java index 6591dd01..c43644bc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java @@ -1,10 +1,11 @@ package com.rarchives.ripme.ripper.rippers; -import java.io.File; import java.io.IOException; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; @@ -330,9 +331,9 @@ public class FivehundredpxRipper extends AbstractJSONRipper { public void downloadURL(URL url, int index) { String u = url.toExternalForm(); String[] fields = u.split("/"); - String prefix = getPrefix(index) + fields[fields.length - 3]; - File saveAs = new File(getWorkingDir() + File.separator + prefix + ".jpg"); - addURLToDownload(url, saveAs, "", null, false); + String prefix = "/" + getPrefix(index) + fields[fields.length - 3]; + Path saveAs = Paths.get(getWorkingDir() + prefix + ".jpg"); + addURLToDownload(url, saveAs.toFile(), "", null, false); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java index 99a066a6..47fa330b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java @@ -1,10 +1,12 @@ package com.rarchives.ripme.ripper.rippers; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -15,7 +17,6 @@ import java.util.regex.Pattern; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.utils.Utils; import org.jsoup.Connection.Response; -import org.jsoup.HttpStatusException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -91,14 +92,13 @@ public class FuraffinityRipper extends AbstractHTMLRipper { String nextUrl = urlBase + nextPageUrl.first().attr("href"); sleep(500); - Document nextPage = Http.url(nextUrl).cookies(cookies).get(); - return nextPage; + return Http.url(nextUrl).cookies(cookies).get(); } private String getImageFromPost(String url) { sleep(1000); - Document d = null; + Document d; try { d = Http.url(url).cookies(cookies).get(); Elements links = d.getElementsByTag("a"); @@ -184,24 +184,22 @@ public class FuraffinityRipper extends AbstractHTMLRipper { } String newText = ""; String saveAs = ""; - File saveFileAs; + Path saveFileAs; saveAs = text.split("\n")[0]; saveAs = saveAs.replaceAll("^(\\S+)\\s+by\\s+(.*)$", "$2_$1"); for (int i = 1;i < text.split("\n").length; i++) { newText = newText.replace("\\","").replace("/","").replace("~","") + "\n" + text.split("\n")[i]; } try { - if (!subdirectory.equals("")) { - subdirectory = File.separator + subdirectory; - } - saveFileAs = new File( - workingDir.getCanonicalPath() + saveFileAs = Paths.get( + workingDir + + "/" + subdirectory - + File.separator + + "/" + saveAs + ".txt"); // Write the file - FileOutputStream out = (new FileOutputStream(saveFileAs)); + OutputStream out = Files.newOutputStream(saveFileAs); out.write(text.getBytes()); out.close(); } catch (IOException e) { @@ -209,9 +207,13 @@ public class FuraffinityRipper extends AbstractHTMLRipper { return false; } LOGGER.debug("Downloading " + url + "'s description to " + saveFileAs); - if (!saveFileAs.getParentFile().exists()) { - LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); - saveFileAs.getParentFile().mkdirs(); + if (!Files.exists(saveFileAs.getParent())) { + LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent().toFile())); + try { + Files.createDirectory(saveFileAs.getParent()); + } catch (IOException e) { + e.printStackTrace(); + } } return true; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java index 288ca94e..4eb69c58 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java @@ -3,12 +3,13 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; -import java.io.File; import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -58,7 +59,7 @@ public class PahealRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document page) throws IOException { for (Element e : page.select("#paginator a")) { - if (e.text().toLowerCase().equals("next")) { + if (e.text().equalsIgnoreCase("next")) { return Http.url(e.absUrl("href")).cookies(getCookies()).get(); } } @@ -90,12 +91,12 @@ public class PahealRipper extends AbstractHTMLRipper { name = name.substring(0, name.length() - ext.length()); } - File outFile = new File(workingDir.getCanonicalPath() - + File.separator + Path outFile = Paths.get(workingDir + + "/" + Utils.filesystemSafe(new URI(name).getPath()) + ext); - addURLToDownload(url, outFile); - } catch (IOException | URISyntaxException ex) { + addURLToDownload(url, outFile.toFile()); + } catch (URISyntaxException ex) { logger.error("Error while downloading URL " + url, ex); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java index 197bdcbd..0c0efd14 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java @@ -1,9 +1,9 @@ package com.rarchives.ripme.ripper.rippers; -import java.io.File; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; @@ -74,7 +74,7 @@ public class PornhubRipper extends AbstractHTMLRipper { @Override protected void downloadURL(URL url, int index) { - PornhubImageThread t = new PornhubImageThread(url, index, this.workingDir); + PornhubImageThread t = new PornhubImageThread(url, index, this.workingDir.toPath()); pornhubThreadPool.addThread(t); try { Thread.sleep(IMAGE_SLEEP_TIME); @@ -130,7 +130,7 @@ public class PornhubRipper extends AbstractHTMLRipper { private URL url; private int index; - PornhubImageThread(URL url, int index, File workingDir) { + PornhubImageThread(URL url, int index, Path workingDir) { super(); this.url = url; this.index = index; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 765f9797..66f4a809 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -1,10 +1,12 @@ package com.rarchives.ripme.ripper.rippers; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Date; import java.util.List; import java.util.regex.Matcher; @@ -13,6 +15,7 @@ import java.util.regex.Pattern; import com.rarchives.ripme.ui.RipStatusMessage; import j2html.TagCreator; import j2html.tags.ContainerTag; +import j2html.tags.specialized.DivTag; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -241,7 +244,7 @@ public class RedditRipper extends AlbumRipper { } private void saveText(JSONArray jsonArray) throws JSONException { - File saveFileAs; + Path saveFileAs; JSONObject selfPost = jsonArray.getJSONObject(0).getJSONObject("data") .getJSONArray("children").getJSONObject(0).getJSONObject("data"); @@ -284,11 +287,11 @@ public class RedditRipper extends AlbumRipper { ).renderFormatted(); try { - saveFileAs = new File(workingDir.getCanonicalPath() - + "" + File.separator + saveFileAs = Paths.get(workingDir + + "/" + id + "_" + title.replaceAll("[\\\\/:*?\"<>|]", "") + ".html"); - FileOutputStream out = new FileOutputStream(saveFileAs); + OutputStream out = Files.newOutputStream(saveFileAs); out.write(html.getBytes()); out.close(); } catch (IOException e) { @@ -298,26 +301,30 @@ public class RedditRipper extends AlbumRipper { LOGGER.debug("Downloading " + url + "'s self post to " + saveFileAs); super.retrievingSource(permalink); - if (!saveFileAs.getParentFile().exists()) { - LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); - saveFileAs.getParentFile().mkdirs(); + if (!Files.exists(saveFileAs.getParent())) { + LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent().toFile())); + try { + Files.createDirectory(saveFileAs.getParent()); + } catch (IOException e) { + e.printStackTrace(); + } } } private ContainerTag getComments(JSONArray comments, String author) { - ContainerTag commentsDiv = div().withId("comments"); + ContainerTag commentsDiv = div().withId("comments"); for (int i = 0; i < comments.length(); i++) { JSONObject data = comments.getJSONObject(i).getJSONObject("data"); - ContainerTag commentDiv = + ContainerTag commentDiv = div( span(data.getString("author")).withClasses("author", iff(data.getString("author").equals(author), "op")), a(new Date((long) data.getInt("created") * 1000).toString()).withHref("#" + data.getString("name")) ).withClass("thing comment").withId(data.getString("name")) .with(rawHtml(Jsoup.parse(data.getString("body_html")).text())); - commentDiv = getNestedComments(data, commentDiv, author); + getNestedComments(data, commentDiv, author); commentsDiv.with(commentDiv); } return commentsDiv; @@ -331,7 +338,7 @@ public class RedditRipper extends AlbumRipper { .getJSONArray("children") .getJSONObject(i).getJSONObject("data"); - ContainerTag childDiv = + ContainerTag childDiv = div( div( span(nestedComment.getString("author")).withClasses("author", iff(nestedComment.getString("author").equals(author), "op")), @@ -347,7 +354,7 @@ public class RedditRipper extends AlbumRipper { } private URL parseRedditVideoMPD(String vidURL) { - org.jsoup.nodes.Document doc = null; + org.jsoup.nodes.Document doc; try { doc = Http.url(vidURL + "/DASHPlaylist.mpd").ignoreContentType().get(); int largestHeight = 0; @@ -395,17 +402,17 @@ public class RedditRipper extends AlbumRipper { Matcher m = p.matcher(url); if (m.matches()) { // It's from reddituploads. Assume .jpg extension. - String savePath = this.workingDir + File.separator; + String savePath = this.workingDir + "/"; savePath += id + "-" + m.group(1) + title + ".jpg"; - addURLToDownload(urls.get(0), new File(savePath)); + addURLToDownload(urls.get(0), Paths.get(savePath).toFile()); } if (url.contains("v.redd.it")) { - String savePath = this.workingDir + File.separator; + String savePath = this.workingDir + "/"; savePath += id + "-" + url.split("/")[3] + title + ".mp4"; URL urlToDownload = parseRedditVideoMPD(urls.get(0).toExternalForm()); if (urlToDownload != null) { LOGGER.info("url: " + urlToDownload + " file: " + savePath); - addURLToDownload(urlToDownload, new File(savePath)); + addURLToDownload(urlToDownload, Paths.get(savePath).toFile()); } } else { @@ -428,7 +435,6 @@ public class RedditRipper extends AlbumRipper { if (Utils.getConfigBoolean("reddit.use_sub_dirs", true)) { if (Utils.getConfigBoolean("album_titles.save", true)) { subdirectory = title; - title = "-" + title + "-"; } } for (int i = 0; i < data.length(); i++) { From 6c74922d6a4b65262d3e68b6b0eaf17f7a3776df Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 19:39:16 +0100 Subject: [PATCH 273/512] ripstatuscomplete java.nio --- .../rarchives/ripme/ripper/AbstractRipper.java | 2 +- .../java/com/rarchives/ripme/ui/MainWindow.java | 8 ++++---- .../rarchives/ripme/ui/RipStatusComplete.java | 16 +++++----------- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 4766d83d..7d7275dd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -486,7 +486,7 @@ public abstract class AbstractRipper completed = true; LOGGER.info(" Rip completed!"); - RipStatusComplete rsc = new RipStatusComplete(workingDir, getCount()); + RipStatusComplete rsc = new RipStatusComplete(workingDir.toPath(), getCount()); RipStatusMessage msg = new RipStatusMessage(STATUS.RIP_COMPLETE, rsc); observer.update(this, msg); diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 430c5460..884df89f 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1440,8 +1440,8 @@ public final class MainWindow implements Runnable, RipStatusHandler { statusProgress.setValue(0); statusProgress.setVisible(false); openButton.setVisible(true); - File f = rsc.dir; - String prettyFile = Utils.shortenPath(f); + Path f = rsc.dir; + String prettyFile = Utils.shortenPath(f.toFile()); openButton.setText(Utils.getLocalizedString("open") + prettyFile); mainFrame.setTitle("RipMe v" + UpdateUtils.getThisJarVersion()); try { @@ -1458,7 +1458,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { try { String commandToRun = Utils.getConfigString("finish.command", "ls"); commandToRun = commandToRun.replaceAll("%url%", url); - commandToRun = commandToRun.replaceAll("%path%", f.getAbsolutePath()); + commandToRun = commandToRun.replaceAll("%path%", f.toAbsolutePath().toString()); LOGGER.info("RUnning command " + commandToRun); // code from: // https://stackoverflow.com/questions/5711084/java-runtime-getruntime-getting-output-from-executing-a-command-line-program @@ -1484,7 +1484,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { LOGGER.error(e.getStackTrace()); } } - appendLog("Rip complete, saved to " + f.getAbsolutePath(), Color.GREEN); + appendLog("Rip complete, saved to " + f, Color.GREEN); openButton.setActionCommand(f.toString()); openButton.addActionListener(event -> { try { diff --git a/src/main/java/com/rarchives/ripme/ui/RipStatusComplete.java b/src/main/java/com/rarchives/ripme/ui/RipStatusComplete.java index 720aa9a7..418da9d0 100644 --- a/src/main/java/com/rarchives/ripme/ui/RipStatusComplete.java +++ b/src/main/java/com/rarchives/ripme/ui/RipStatusComplete.java @@ -1,29 +1,23 @@ package com.rarchives.ripme.ui; -import java.io.File; import java.io.IOException; +import java.nio.file.Path; public class RipStatusComplete { - File dir = null; + Path dir = null; int count = 0; - public RipStatusComplete(File dir) { + public RipStatusComplete(Path dir) { this.dir = dir; this.count = 1; } - public RipStatusComplete(File dir, int count) { + public RipStatusComplete(Path dir, int count) { this.dir = dir; this.count = count; } public String getDir() { - String result; - try { - result = this.dir.getCanonicalPath(); - } catch (IOException e) { - result = this.dir.toString(); - } - return result; + return this.dir.toString(); } } From 01e6b5b0d01f4e78475cdba611e9f85e2fe6e490 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 19:46:55 +0100 Subject: [PATCH 274/512] videoripper java.nio --- src/main/java/com/rarchives/ripme/ripper/VideoRipper.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 9e935d62..8922b93f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -44,10 +44,11 @@ public abstract class VideoRipper extends AbstractRipper { } @Override - public boolean addURLToDownload(URL url, File saveAs) { + public boolean addURLToDownload(URL url, File saveAsF) { + Path saveAs = saveAsF.toPath(); if (Utils.getConfigBoolean("urls_only.save", false)) { // Output URL to file - String urlFile = this.workingDir + File.separator + "urls.txt"; + String urlFile = this.workingDir + "/urls.txt"; try (FileWriter fw = new FileWriter(urlFile, true)) { fw.write(url.toExternalForm()); From a3df3d7157560bd3751c7f945979e0617a39990d Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 19:56:08 +0100 Subject: [PATCH 275/512] downloadvideothread java.nio --- .../ripme/ripper/DownloadVideoThread.java | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index f8b4b087..69734292 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -1,13 +1,13 @@ package com.rarchives.ripme.ripper; import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; import javax.net.ssl.HttpsURLConnection; @@ -25,16 +25,16 @@ class DownloadVideoThread extends Thread { private static final Logger logger = LogManager.getLogger(DownloadVideoThread.class); private URL url; - private File saveAs; + private Path saveAs; private String prettySaveAs; private AbstractRipper observer; private int retries; - public DownloadVideoThread(URL url, File saveAs, AbstractRipper observer) { + public DownloadVideoThread(URL url, Path saveAs, AbstractRipper observer) { super(); this.url = url; this.saveAs = saveAs; - this.prettySaveAs = Utils.removeCWD(saveAs); + this.prettySaveAs = Utils.removeCWD(saveAs.toFile()); this.observer = observer; this.retries = Utils.getConfigInteger("download.retries", 1); } @@ -50,13 +50,17 @@ class DownloadVideoThread extends Thread { observer.downloadErrored(url, "Download interrupted"); return; } - if (saveAs.exists()) { + if (Files.exists(saveAs)) { if (Utils.getConfigBoolean("file.overwrite", false)) { logger.info("[!] Deleting existing file" + prettySaveAs); - saveAs.delete(); + try { + Files.delete(saveAs); + } catch (IOException e) { + e.printStackTrace(); + } } else { logger.info("[!] Skipping " + url + " -- file already exists: " + prettySaveAs); - observer.downloadExists(url, saveAs); + observer.downloadExists(url, saveAs.toFile()); return; } } @@ -100,7 +104,7 @@ class DownloadVideoThread extends Thread { huc.connect(); // Check status code bis = new BufferedInputStream(huc.getInputStream()); - fos = new FileOutputStream(saveAs); + fos = Files.newOutputStream(saveAs); while ( (bytesRead = bis.read(data)) != -1) { try { observer.stopCheck(); @@ -122,10 +126,10 @@ class DownloadVideoThread extends Thread { // Close any open streams try { if (bis != null) { bis.close(); } - } catch (IOException e) { } + } catch (IOException ignored) { } try { if (fos != null) { fos.close(); } - } catch (IOException e) { } + } catch (IOException ignored) { } } if (tries > this.retries) { logger.error("[!] Exceeded maximum retries (" + this.retries + ") for URL " + url); @@ -133,7 +137,7 @@ class DownloadVideoThread extends Thread { return; } } while (true); - observer.downloadCompleted(url, saveAs); + observer.downloadCompleted(url, saveAs.toFile()); logger.info("[+] Saved " + url + " as " + this.prettySaveAs); } From 1422e32a43155ea38424239ec948e33687ea7441 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 20:10:31 +0100 Subject: [PATCH 276/512] downloadexists java.nio --- .../java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 6 +++--- .../java/com/rarchives/ripme/ripper/AbstractJSONRipper.java | 6 +++--- .../java/com/rarchives/ripme/ripper/AbstractRipper.java | 3 ++- src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java | 6 +++--- .../java/com/rarchives/ripme/ripper/DownloadFileThread.java | 2 +- .../com/rarchives/ripme/ripper/DownloadVideoThread.java | 2 +- src/main/java/com/rarchives/ripme/ripper/VideoRipper.java | 5 ++--- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 5afaf936..c6a8c3df 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -403,14 +403,14 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { Tells user that a single file in the album they wish to download has already been downloaded in the past. */ - public void downloadExists(URL url, File file) { + public void downloadExists(URL url, Path file) { if (observer == null) { return; } itemsPending.remove(url); - itemsCompleted.put(url, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file.getAbsolutePath())); + itemsCompleted.put(url, file.toFile()); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); checkIfComplete(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 6be88472..7bcab02c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -242,14 +242,14 @@ public abstract class AbstractJSONRipper extends AbstractRipper { * Tells user that a single file in the album they wish to download has * already been downloaded in the past. */ - public void downloadExists(URL url, File file) { + public void downloadExists(URL url, Path file) { if (observer == null) { return; } itemsPending.remove(url); - itemsCompleted.put(url, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file.getAbsolutePath())); + itemsCompleted.put(url, file.toFile()); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); checkIfComplete(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 7d7275dd..2606068b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -9,6 +9,7 @@ import java.io.IOException; import java.lang.reflect.Constructor; import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -464,7 +465,7 @@ public abstract class AbstractRipper * @param url * @param file */ - public abstract void downloadExists(URL url, File file); + public abstract void downloadExists(URL url, Path file); /** * @return Number of files downloaded. diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 554c0e54..094fcb10 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -151,14 +151,14 @@ public abstract class AlbumRipper extends AbstractRipper { * Tells user that a single file in the album they wish to download has * already been downloaded in the past. */ - public void downloadExists(URL url, File file) { + public void downloadExists(URL url, Path file) { if (observer == null) { return; } itemsPending.remove(url); - itemsCompleted.put(url, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file.getAbsolutePath())); + itemsCompleted.put(url, file.toFile()); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); checkIfComplete(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index f879e069..983b931a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -89,7 +89,7 @@ class DownloadFileThread extends Thread { } else { logger.info("[!] " + Utils.getLocalizedString("skipping") + " " + url + " -- " + Utils.getLocalizedString("file.already.exists") + ": " + prettySaveAs); - observer.downloadExists(url, saveAs); + observer.downloadExists(url, saveAs.toPath()); return; } } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index 69734292..c51c9906 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -60,7 +60,7 @@ class DownloadVideoThread extends Thread { } } else { logger.info("[!] Skipping " + url + " -- file already exists: " + prettySaveAs); - observer.downloadExists(url, saveAs.toFile()); + observer.downloadExists(url, saveAs); return; } } diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 8922b93f..0499354e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -153,12 +153,11 @@ public abstract class VideoRipper extends AbstractRipper { /** * Runs if user tries to redownload an already existing File. - * - * @param url Target URL + * @param url Target URL * @param file Existing file */ @Override - public void downloadExists(URL url, File file) { + public void downloadExists(URL url, Path file) { if (observer == null) { return; } From 901678224857597375610b2c3e6aec74ca84dd02 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 20:19:41 +0100 Subject: [PATCH 277/512] addURLToDownload(url, saveAs) java.nio --- .../java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 4 ++-- .../java/com/rarchives/ripme/ripper/AbstractJSONRipper.java | 4 ++-- src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java | 2 +- src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java | 4 ++-- src/main/java/com/rarchives/ripme/ripper/VideoRipper.java | 5 ++--- .../com/rarchives/ripme/ripper/rippers/EHentaiRipper.java | 2 +- .../java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java | 2 +- .../com/rarchives/ripme/ripper/rippers/PahealRipper.java | 2 +- .../com/rarchives/ripme/ripper/rippers/RedditRipper.java | 4 ++-- 9 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index c6a8c3df..aa84cd67 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -345,8 +345,8 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { } @Override - public boolean addURLToDownload(URL url, File saveAs) { - return addURLToDownload(url, saveAs, null, null, false); + public boolean addURLToDownload(URL url, Path saveAs) { + return addURLToDownload(url, saveAs.toFile(), null, null, false); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 7bcab02c..4db46f8b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -184,8 +184,8 @@ public abstract class AbstractJSONRipper extends AbstractRipper { } @Override - public boolean addURLToDownload(URL url, File saveAs) { - return addURLToDownload(url, saveAs, null, null, false); + public boolean addURLToDownload(URL url, Path saveAs) { + return addURLToDownload(url, saveAs.toFile(), null, null, false); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 2606068b..5a4e6654 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -208,7 +208,7 @@ public abstract class AbstractRipper * Path of the local file to save the content to. * @return True on success, false on failure. */ - public abstract boolean addURLToDownload(URL url, File saveAs); + public abstract boolean addURLToDownload(URL url, Path saveAs); /** * Queues image to be downloaded and saved. diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 094fcb10..3342c3c8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -93,8 +93,8 @@ public abstract class AlbumRipper extends AbstractRipper { } @Override - public boolean addURLToDownload(URL url, File saveAs) { - return addURLToDownload(url, saveAs, null, null, false); + public boolean addURLToDownload(URL url, Path saveAs) { + return addURLToDownload(url, saveAs.toFile(), null, null, false); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 0499354e..dc9cbe09 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -44,8 +44,7 @@ public abstract class VideoRipper extends AbstractRipper { } @Override - public boolean addURLToDownload(URL url, File saveAsF) { - Path saveAs = saveAsF.toPath(); + public boolean addURLToDownload(URL url, Path saveAs) { if (Utils.getConfigBoolean("urls_only.save", false)) { // Output URL to file String urlFile = this.workingDir + "/urls.txt"; @@ -75,7 +74,7 @@ public abstract class VideoRipper extends AbstractRipper { @Override public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - return addURLToDownload(url, saveAs); + return addURLToDownload(url, saveAs.toPath()); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java index 13568758..ba7e446d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java @@ -249,7 +249,7 @@ public class EHentaiRipper extends AbstractHTMLRipper { savePath += String.format("%03d_", index); } savePath += m.group(1); - addURLToDownload(new URL(imgsrc), Paths.get(savePath).toFile()); + addURLToDownload(new URL(imgsrc), Paths.get(savePath)); } else { // Provide prefix and let the AbstractRipper "guess" the filename String prefix = ""; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index 93cb809e..4a049b93 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -230,7 +230,7 @@ public class ImgurRipper extends AlbumRipper { } saveAs += imgurImage.getSaveAs(); saveAs = saveAs.replaceAll("\\?\\d", ""); - addURLToDownload(imgurImage.url, new File(saveAs)); + addURLToDownload(imgurImage.url, new File(saveAs).toPath()); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java index 4eb69c58..39d56b83 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java @@ -95,7 +95,7 @@ public class PahealRipper extends AbstractHTMLRipper { + "/" + Utils.filesystemSafe(new URI(name).getPath()) + ext); - addURLToDownload(url, outFile.toFile()); + addURLToDownload(url, outFile); } catch (URISyntaxException ex) { logger.error("Error while downloading URL " + url, ex); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 66f4a809..8947f4de 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -404,7 +404,7 @@ public class RedditRipper extends AlbumRipper { // It's from reddituploads. Assume .jpg extension. String savePath = this.workingDir + "/"; savePath += id + "-" + m.group(1) + title + ".jpg"; - addURLToDownload(urls.get(0), Paths.get(savePath).toFile()); + addURLToDownload(urls.get(0), Paths.get(savePath)); } if (url.contains("v.redd.it")) { String savePath = this.workingDir + "/"; @@ -412,7 +412,7 @@ public class RedditRipper extends AlbumRipper { URL urlToDownload = parseRedditVideoMPD(urls.get(0).toExternalForm()); if (urlToDownload != null) { LOGGER.info("url: " + urlToDownload + " file: " + savePath); - addURLToDownload(urlToDownload, Paths.get(savePath).toFile()); + addURLToDownload(urlToDownload, Paths.get(savePath)); } } else { From 3c89ca8428990c71a2a3700b30522e00a9093200 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 19:32:40 +0100 Subject: [PATCH 278/512] imgur java.nio --- .../ripme/ripper/rippers/ImgurRipper.java | 99 ++++++------------- 1 file changed, 32 insertions(+), 67 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index 4a049b93..10c633ba 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -1,9 +1,10 @@ package com.rarchives.ripme.ripper.rippers; -import java.io.File; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; @@ -91,7 +92,7 @@ public class ImgurRipper extends AlbumRipper { albumDoc = Http.url(url).get(); } - Elements elems = null; + Elements elems; /* // TODO: Add config option for including username in album title. @@ -106,15 +107,13 @@ public class ImgurRipper extends AlbumRipper { } */ - String title = null; + String title; final String defaultTitle1 = "Imgur: The most awesome images on the Internet"; final String defaultTitle2 = "Imgur: The magic of the Internet"; LOGGER.info("Trying to get album title"); elems = albumDoc.select("meta[property=og:title]"); - if (elems != null) { - title = elems.attr("content"); - LOGGER.debug("Title is " + title); - } + title = elems.attr("content"); + LOGGER.debug("Title is " + title); // This is here encase the album is unnamed, to prevent // Imgur: The most awesome images on the Internet from being added onto the album name if (title.contains(defaultTitle1) || title.contains(defaultTitle2)) { @@ -124,27 +123,17 @@ public class ImgurRipper extends AlbumRipper { title = ""; LOGGER.debug("Trying to use title tag to get title"); elems = albumDoc.select("title"); - if (elems != null) { - if (elems.text().contains(defaultTitle1) || elems.text().contains(defaultTitle2)) { - LOGGER.debug("Was unable to get album title or album was untitled"); - } - else { - title = elems.text(); - } + if (elems.text().contains(defaultTitle1) || elems.text().contains(defaultTitle2)) { + LOGGER.debug("Was unable to get album title or album was untitled"); + } + else { + title = elems.text(); } } String albumTitle = "imgur_"; - /* - // TODO: Add config option (see above) - if (user != null) { - albumTitle += "user_" + user; - } - */ albumTitle += gid; - if (title != null) { - albumTitle += "_" + title; - } + albumTitle += "_" + title; return albumTitle; } catch (IOException e) { @@ -204,33 +193,25 @@ public class ImgurRipper extends AlbumRipper { } private void ripAlbum(URL url, String subdirectory) throws IOException { - int index = 0; + int index; this.sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm()); index = 0; ImgurAlbum album = getImgurAlbum(url); for (ImgurImage imgurImage : album.images) { stopCheck(); - String saveAs = workingDir.getCanonicalPath(); - if (!saveAs.endsWith(File.separator)) { - saveAs += File.separator; - } + Path saveAs = workingDir.toPath(); if (subdirectory != null && !subdirectory.equals("")) { - saveAs += subdirectory; + saveAs.resolve(subdirectory); } - if (!saveAs.endsWith(File.separator)) { - saveAs += File.separator; - } - File subdirFile = new File(saveAs); - if (!subdirFile.exists()) { - subdirFile.mkdirs(); + if (!Files.exists(saveAs)) { + Files.createDirectory(saveAs); } index += 1; if (Utils.getConfigBoolean("download.save_order", true)) { - saveAs += String.format("%03d_", index); + saveAs.resolve(String.format("%03d_", index)); } - saveAs += imgurImage.getSaveAs(); - saveAs = saveAs.replaceAll("\\?\\d", ""); - addURLToDownload(imgurImage.url, new File(saveAs).toPath()); + saveAs.resolve(imgurImage.getSaveAs().replaceAll("\\?\\d", "")); + addURLToDownload(imgurImage.url, saveAs); } } @@ -309,7 +290,7 @@ public class ImgurRipper extends AlbumRipper { image = "http:" + thumb.select("img").attr("src"); } else { // Unable to find image in this div - LOGGER.error("[!] Unable to find image in div: " + thumb.toString()); + LOGGER.error("[!] Unable to find image in div: " + thumb); continue; } if (image.endsWith(".gif") && Utils.getConfigBoolean("prefer.mp4", false)) { @@ -336,10 +317,6 @@ public class ImgurRipper extends AlbumRipper { return imgurAlbum; } - private static ImgurImage createImgurImageFromJson(JSONObject json) throws MalformedURLException { - return new ImgurImage(extractImageUrlFromJson(json)); - } - private static URL extractImageUrlFromJson(JSONObject json) throws MalformedURLException { String ext = json.getString("ext"); if (ext.equals(".gif") && Utils.getConfigBoolean("prefer.mp4", false)) { @@ -374,7 +351,6 @@ public class ImgurRipper extends AlbumRipper { * Rips all albums in an imgur user's account. * @param url * URL to imgur user account (http://username.imgur.com) - * @throws IOException */ private void ripUserAccount(URL url) throws IOException { LOGGER.info("Retrieving " + url); @@ -390,14 +366,14 @@ public class ImgurRipper extends AlbumRipper { URL albumURL = new URL("http:" + album.attr("href") + "/noscript"); try { ripAlbum(albumURL, albumID); - Thread.sleep(SLEEP_BETWEEN_ALBUMS * 1000); + Thread.sleep(SLEEP_BETWEEN_ALBUMS * 1000L); } catch (Exception e) { LOGGER.error("Error while ripping album: " + e.getMessage(), e); } } } - private void ripUserImages(URL url) throws IOException { + private void ripUserImages(URL url) { int page = 0; int imagesFound = 0; int imagesTotal = 0; String jsonUrl = url.toExternalForm().replace("/all", "/ajax/images"); if (jsonUrl.contains("#")) { @@ -478,8 +454,8 @@ public class ImgurRipper extends AlbumRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = null; - Matcher m = null; + Pattern p; + Matcher m; p = Pattern.compile("^https?://(www\\.|m\\.)?imgur\\.com/(a|gallery)/([a-zA-Z0-9]{5,}).*$"); m = p.matcher(url.toExternalForm()); @@ -529,13 +505,13 @@ public class ImgurRipper extends AlbumRipper { if (m.matches()) { // Imgur subreddit aggregator albumType = ALBUM_TYPE.SUBREDDIT; - String album = m.group(2); + StringBuilder album = new StringBuilder(m.group(2)); for (int i = 3; i <= m.groupCount(); i++) { if (m.group(i) != null) { - album += "_" + m.group(i).replace("/", ""); + album.append("_").append(m.group(i).replace("/", "")); } } - return album; + return album.toString(); } p = Pattern.compile("^https?://(i\\.|www\\.|m\\.)?imgur\\.com/r/(\\w+)/([a-zA-Z0-9,]{5,}).*$"); m = p.matcher(url.toExternalForm()); @@ -568,15 +544,11 @@ public class ImgurRipper extends AlbumRipper { throw new MalformedURLException("Unsupported imgur URL format: " + url.toExternalForm()); } - public ALBUM_TYPE getAlbumType() { - return albumType; - } - public static class ImgurImage { String title = ""; String description = ""; - String extension = ""; - public URL url = null; + String extension; + public URL url; ImgurImage(URL url) { this.url = url; @@ -586,14 +558,7 @@ public class ImgurRipper extends AlbumRipper { this.extension = this.extension.substring(0, this.extension.indexOf("?")); } } - ImgurImage(URL url, String title) { - this(url); - this.title = title; - } - public ImgurImage(URL url, String title, String description) { - this(url, title); - this.description = description; - } + String getSaveAs() { String saveAs = this.title; String u = url.toExternalForm(); @@ -613,7 +578,7 @@ public class ImgurRipper extends AlbumRipper { public static class ImgurAlbum { String title = null; - public URL url = null; + public URL url; public List images = new ArrayList<>(); ImgurAlbum(URL url) { this.url = url; From 2171d287f869b8f67ee1aa254062a47262e8fb0c Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 20:36:35 +0100 Subject: [PATCH 279/512] addURLToDownload(url, saveAs, referrer, cookies, getFileExtFromMIME) java.nio --- .../com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 10 +++++----- .../com/rarchives/ripme/ripper/AbstractJSONRipper.java | 10 +++++----- .../com/rarchives/ripme/ripper/AbstractRipper.java | 4 ++-- .../java/com/rarchives/ripme/ripper/AlbumRipper.java | 10 +++++----- .../java/com/rarchives/ripme/ripper/VideoRipper.java | 4 ++-- .../ripme/ripper/rippers/FivehundredpxRipper.java | 2 +- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index aa84cd67..83b306b4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -303,7 +303,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { /* Queues multiple URLs of single images to download from a single Album URL */ - public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { + public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { // Only download one file if this is a test. if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { stop(); @@ -315,7 +315,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { || itemsCompleted.containsKey(url) || itemsErrored.containsKey(url) )) { // Item is already downloaded/downloading, skip it. - LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); + LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs.toFile())); return false; } if (Utils.getConfigBoolean("urls_only.save", false)) { @@ -330,8 +330,8 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { } } else { - itemsPending.put(url, saveAs); - DownloadFileThread dft = new DownloadFileThread(url, saveAs, this, getFileExtFromMIME); + itemsPending.put(url, saveAs.toFile()); + DownloadFileThread dft = new DownloadFileThread(url, saveAs.toFile(), this, getFileExtFromMIME); if (referrer != null) { dft.setReferrer(referrer); } @@ -346,7 +346,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { @Override public boolean addURLToDownload(URL url, Path saveAs) { - return addURLToDownload(url, saveAs.toFile(), null, null, false); + return addURLToDownload(url, saveAs, null, null, false); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 4db46f8b..76c08905 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -142,7 +142,7 @@ public abstract class AbstractJSONRipper extends AbstractRipper { /** * Queues multiple URLs of single images to download from a single Album URL */ - public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { + public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { // Only download one file if this is a test. if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { stop(); @@ -154,7 +154,7 @@ public abstract class AbstractJSONRipper extends AbstractRipper { || itemsCompleted.containsKey(url) || itemsErrored.containsKey(url) )) { // Item is already downloaded/downloading, skip it. - LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); + LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs.toFile())); return false; } if (Utils.getConfigBoolean("urls_only.save", false)) { @@ -169,8 +169,8 @@ public abstract class AbstractJSONRipper extends AbstractRipper { } } else { - itemsPending.put(url, saveAs); - DownloadFileThread dft = new DownloadFileThread(url, saveAs, this, getFileExtFromMIME); + itemsPending.put(url, saveAs.toFile()); + DownloadFileThread dft = new DownloadFileThread(url, saveAs.toFile(), this, getFileExtFromMIME); if (referrer != null) { dft.setReferrer(referrer); } @@ -185,7 +185,7 @@ public abstract class AbstractJSONRipper extends AbstractRipper { @Override public boolean addURLToDownload(URL url, Path saveAs) { - return addURLToDownload(url, saveAs.toFile(), null, null, false); + return addURLToDownload(url, saveAs, null, null, false); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 5a4e6654..d1267adf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -224,7 +224,7 @@ public abstract class AbstractRipper * True if downloaded successfully * False if failed to download */ - protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, + protected abstract boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME); /** @@ -350,7 +350,7 @@ public abstract class AbstractRipper LOGGER.debug("Unable to write URL history file"); } } - return addURLToDownload(url, saveFileAs, referrer, cookies, getFileExtFromMIME); + return addURLToDownload(url, saveFileAs.toPath(), referrer, cookies, getFileExtFromMIME); } protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName, String extension) { diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 3342c3c8..cb2c62da 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -51,7 +51,7 @@ public abstract class AlbumRipper extends AbstractRipper { /** * Queues multiple URLs of single images to download from a single Album URL */ - public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { + public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { // Only download one file if this is a test. if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { stop(); @@ -63,7 +63,7 @@ public abstract class AlbumRipper extends AbstractRipper { || itemsCompleted.containsKey(url) || itemsErrored.containsKey(url) )) { // Item is already downloaded/downloading, skip it. - LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); + LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs.toFile())); return false; } if (Utils.getConfigBoolean("urls_only.save", false)) { @@ -78,8 +78,8 @@ public abstract class AlbumRipper extends AbstractRipper { } } else { - itemsPending.put(url, saveAs); - DownloadFileThread dft = new DownloadFileThread(url, saveAs, this, getFileExtFromMIME); + itemsPending.put(url, saveAs.toFile()); + DownloadFileThread dft = new DownloadFileThread(url, saveAs.toFile(), this, getFileExtFromMIME); if (referrer != null) { dft.setReferrer(referrer); } @@ -94,7 +94,7 @@ public abstract class AlbumRipper extends AbstractRipper { @Override public boolean addURLToDownload(URL url, Path saveAs) { - return addURLToDownload(url, saveAs.toFile(), null, null, false); + return addURLToDownload(url, saveAs, null, null, false); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index dc9cbe09..5592f909 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -73,8 +73,8 @@ public abstract class VideoRipper extends AbstractRipper { } @Override - public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - return addURLToDownload(url, saveAs.toPath()); + public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { + return addURLToDownload(url, saveAs); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java index c43644bc..79edab1c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java @@ -333,7 +333,7 @@ public class FivehundredpxRipper extends AbstractJSONRipper { String[] fields = u.split("/"); String prefix = "/" + getPrefix(index) + fields[fields.length - 3]; Path saveAs = Paths.get(getWorkingDir() + prefix + ".jpg"); - addURLToDownload(url, saveAs.toFile(), "", null, false); + addURLToDownload(url, saveAs, "", null, false); } } From 88fc69a4ddddd4b04998ce07cc8bce52e28fec5b Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 21:39:51 +0100 Subject: [PATCH 280/512] imgur flaky --- .../com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java index 37ef50eb..28f18bff 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java @@ -6,6 +6,7 @@ import com.rarchives.ripme.utils.RipUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -35,6 +36,7 @@ public class ImgurRipperTest extends RippersTest { } @Test + @Tag("flaky") public void testImgurAlbums() throws IOException { List contentURLs = new ArrayList<>(); // URLs that should return more than 1 image From 65eed02bf20d9e68047b2f5bc93c3137a3c7c8fd Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 20:49:56 +0100 Subject: [PATCH 281/512] Utils.removeCWD java.nio --- .../ripme/ripper/AbstractHTMLRipper.java | 6 ++--- .../ripme/ripper/AbstractJSONRipper.java | 25 +++++++------------ .../rarchives/ripme/ripper/AlbumRipper.java | 6 ++--- .../ripme/ripper/DownloadFileThread.java | 2 +- .../ripme/ripper/DownloadVideoThread.java | 2 +- .../rarchives/ripme/ripper/VideoRipper.java | 4 +-- .../ripper/rippers/FuraffinityRipper.java | 2 +- .../ripme/ripper/rippers/RedditRipper.java | 2 +- .../java/com/rarchives/ripme/utils/Utils.java | 15 +++-------- 9 files changed, 25 insertions(+), 39 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 83b306b4..f5d81f72 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -315,7 +315,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { || itemsCompleted.containsKey(url) || itemsErrored.containsKey(url) )) { // Item is already downloaded/downloading, skip it. - LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs.toFile())); + LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); return false; } if (Utils.getConfigBoolean("urls_only.save", false)) { @@ -371,7 +371,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { return; } try { - String path = Utils.removeCWD(saveAs); + String path = Utils.removeCWD(saveAs.toPath()); RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); itemsPending.remove(url); itemsCompleted.put(url, saveAs); @@ -457,7 +457,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { this.workingDir = new File(path); if (!this.workingDir.exists()) { - LOGGER.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir)); + LOGGER.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir.toPath())); if (!this.workingDir.mkdirs()) { throw new IOException("Failed creating dir: \"" + this.workingDir + "\""); } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 76c08905..c0ecf582 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -10,7 +10,9 @@ import java.io.FileWriter; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -154,7 +156,7 @@ public abstract class AbstractJSONRipper extends AbstractRipper { || itemsCompleted.containsKey(url) || itemsErrored.containsKey(url) )) { // Item is already downloaded/downloading, skip it. - LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs.toFile())); + LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); return false; } if (Utils.getConfigBoolean("urls_only.save", false)) { @@ -210,7 +212,7 @@ public abstract class AbstractJSONRipper extends AbstractRipper { return; } try { - String path = Utils.removeCWD(saveAs); + String path = Utils.removeCWD(saveAs.toPath()); RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); itemsPending.remove(url); itemsCompleted.put(url, saveAs); @@ -277,11 +279,6 @@ public abstract class AbstractJSONRipper extends AbstractRipper { @Override public void setWorkingDir(URL url) throws IOException { Path wd = Utils.getWorkingDirectory(); - // TODO - change to nio - String path = wd.toAbsolutePath().toString(); - if (!path.endsWith(File.separator)) { - path += File.separator; - } String title; if (Utils.getConfigBoolean("album_titles.save", true)) { title = getAlbumTitle(this.url); @@ -291,15 +288,11 @@ public abstract class AbstractJSONRipper extends AbstractRipper { LOGGER.debug("Using album title '" + title + "'"); title = Utils.filesystemSafe(title); - path += title; - path = Utils.getOriginalDirectory(path) + File.separator; // check for case sensitive (unix only) - - this.workingDir = new File(path); - if (!this.workingDir.exists()) { - LOGGER.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir)); - if (!this.workingDir.mkdirs()) { - throw new IOException("Failed creating dir: \"" + this.workingDir + "\""); - } + wd = wd.resolve(title); + if (!Files.exists(wd)) { + LOGGER.info("[+] Creating directory: " + Utils.removeCWD(wd)); + Files.createDirectory(wd); + this.workingDir = wd.toFile(); } LOGGER.debug("Set working directory to: " + this.workingDir); } diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index cb2c62da..5f7e4588 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -63,7 +63,7 @@ public abstract class AlbumRipper extends AbstractRipper { || itemsCompleted.containsKey(url) || itemsErrored.containsKey(url) )) { // Item is already downloaded/downloading, skip it. - LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs.toFile())); + LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); return false; } if (Utils.getConfigBoolean("urls_only.save", false)) { @@ -119,7 +119,7 @@ public abstract class AlbumRipper extends AbstractRipper { return; } try { - String path = Utils.removeCWD(saveAs); + String path = Utils.removeCWD(saveAs.toPath()); RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); itemsPending.remove(url); itemsCompleted.put(url, saveAs); @@ -205,7 +205,7 @@ public abstract class AlbumRipper extends AbstractRipper { this.workingDir = new File(path); if (!this.workingDir.exists()) { - LOGGER.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir)); + LOGGER.info("[+] Creating directory: " + Utils.removeCWD(this.workingDir.toPath())); this.workingDir.mkdirs(); } LOGGER.debug("Set working directory to: " + this.workingDir); diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 983b931a..82fd35d5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -45,7 +45,7 @@ class DownloadFileThread extends Thread { super(); this.url = url; this.saveAs = saveAs; - this.prettySaveAs = Utils.removeCWD(saveAs); + this.prettySaveAs = Utils.removeCWD(saveAs.toPath()); this.observer = observer; this.retries = Utils.getConfigInteger("download.retries", 1); this.TIMEOUT = Utils.getConfigInteger("download.timeout", 60000); diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index c51c9906..df518d4d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -34,7 +34,7 @@ class DownloadVideoThread extends Thread { super(); this.url = url; this.saveAs = saveAs; - this.prettySaveAs = Utils.removeCWD(saveAs.toFile()); + this.prettySaveAs = Utils.removeCWD(saveAs); this.observer = observer; this.retries = Utils.getConfigInteger("download.retries", 1); } diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 5592f909..001a42d1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -96,7 +96,7 @@ public abstract class VideoRipper extends AbstractRipper { workingDir = new File(path); if (!workingDir.exists()) { - LOGGER.info("[+] Creating directory: " + Utils.removeCWD(workingDir)); + LOGGER.info("[+] Creating directory: " + Utils.removeCWD(workingDir.toPath())); workingDir.mkdirs(); } @@ -124,7 +124,7 @@ public abstract class VideoRipper extends AbstractRipper { } try { - String path = Utils.removeCWD(saveAs); + String path = Utils.removeCWD(saveAs.toPath()); RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); observer.update(this, msg); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java index 47fa330b..d3357e9e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java @@ -208,7 +208,7 @@ public class FuraffinityRipper extends AbstractHTMLRipper { } LOGGER.debug("Downloading " + url + "'s description to " + saveFileAs); if (!Files.exists(saveFileAs.getParent())) { - LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent().toFile())); + LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); try { Files.createDirectory(saveFileAs.getParent()); } catch (IOException e) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 8947f4de..da66b536 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -302,7 +302,7 @@ public class RedditRipper extends AlbumRipper { LOGGER.debug("Downloading " + url + "'s self post to " + saveFileAs); super.retrievingSource(permalink); if (!Files.exists(saveFileAs.getParent())) { - LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent().toFile())); + LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); try { Files.createDirectory(saveFileAs.getParent()); } catch (IOException e) { diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 6edcc71e..7a230ed1 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -320,15 +320,8 @@ public class Utils { * @param saveAs The File path * @return saveAs in relation to the CWD */ - public static String removeCWD(File saveAs) { - String prettySaveAs = saveAs.toString(); - try { - prettySaveAs = saveAs.getCanonicalPath(); - String cwd = new File(".").getCanonicalPath() + File.separator; - prettySaveAs = prettySaveAs.replace(cwd, "." + File.separator); - } catch (Exception e) { - LOGGER.error("Exception: ", e); - } + public static String removeCWD(Path saveAs) { + String prettySaveAs = saveAs.relativize(Paths.get(".").toAbsolutePath()).toString(); return prettySaveAs; } @@ -371,7 +364,7 @@ public class Utils { * @return 'file' without the leading current working directory */ public static String removeCWD(String file) { - return removeCWD(new File(file)); + return removeCWD(Paths.get(file)); } /** @@ -465,7 +458,7 @@ public class Utils { * @return The simplified path to the file. */ public static String shortenPath(File file) { - String path = removeCWD(file); + String path = removeCWD(file.toPath()); if (path.length() < SHORTENED_PATH_LENGTH * 2) { return path; } From 5d5ce9d0ddb52112d7dbb1f676caaeb3a9ab56e0 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 21:15:44 +0100 Subject: [PATCH 282/512] downloadCompleted java.nio --- .../ripme/ripper/AbstractHTMLRipper.java | 22 ++++++---- .../ripme/ripper/AbstractJSONRipper.java | 21 +++++----- .../ripme/ripper/AbstractRipper.java | 2 +- .../rarchives/ripme/ripper/AlbumRipper.java | 41 ++++++++++--------- .../ripme/ripper/DownloadFileThread.java | 2 +- .../ripme/ripper/DownloadVideoThread.java | 2 +- .../rarchives/ripme/ripper/VideoRipper.java | 4 +- 7 files changed, 51 insertions(+), 43 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index f5d81f72..951562e5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -6,7 +6,11 @@ import java.io.FileWriter; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -26,7 +30,7 @@ import com.rarchives.ripme.ui.RipStatusMessage; public abstract class AbstractHTMLRipper extends AbstractRipper { private final Map itemsPending = Collections.synchronizedMap(new HashMap<>()); - private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); + private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); protected AbstractHTMLRipper(URL url) throws IOException { @@ -320,11 +324,11 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { } if (Utils.getConfigBoolean("urls_only.save", false)) { // Output URL to file - String urlFile = this.workingDir + File.separator + "urls.txt"; - try (FileWriter fw = new FileWriter(urlFile, true)) { - fw.write(url.toExternalForm()); - fw.write(System.lineSeparator()); - itemsCompleted.put(url, new File(urlFile)); + Path urlFile = Paths.get(this.workingDir + "/urls.txt"); + String text = url.toExternalForm() + System.lineSeparator(); + try { + Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + itemsCompleted.put(url, urlFile); } catch (IOException e) { LOGGER.error("Error while writing to " + urlFile, e); } @@ -366,12 +370,12 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { /* Cleans up & tells user about successful download */ - public void downloadCompleted(URL url, File saveAs) { + public void downloadCompleted(URL url, Path saveAs) { if (observer == null) { return; } try { - String path = Utils.removeCWD(saveAs.toPath()); + String path = Utils.removeCWD(saveAs); RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); itemsPending.remove(url); itemsCompleted.put(url, saveAs); @@ -409,7 +413,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { } itemsPending.remove(url); - itemsCompleted.put(url, file.toFile()); + itemsCompleted.put(url, file); observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); checkIfComplete(); diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index c0ecf582..021e5c64 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -6,13 +6,14 @@ import com.rarchives.ripme.utils.Utils; import org.json.JSONObject; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -24,7 +25,7 @@ import java.util.Map; public abstract class AbstractJSONRipper extends AbstractRipper { private Map itemsPending = Collections.synchronizedMap(new HashMap()); - private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); + private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); private Map itemsErrored = Collections.synchronizedMap(new HashMap()); protected AbstractJSONRipper(URL url) throws IOException { @@ -161,11 +162,11 @@ public abstract class AbstractJSONRipper extends AbstractRipper { } if (Utils.getConfigBoolean("urls_only.save", false)) { // Output URL to file - String urlFile = this.workingDir + File.separator + "urls.txt"; - try (FileWriter fw = new FileWriter(urlFile, true)) { - fw.write(url.toExternalForm()); - fw.write(System.lineSeparator()); - itemsCompleted.put(url, new File(urlFile)); + Path urlFile = Paths.get(this.workingDir + "/urls.txt"); + String text = url.toExternalForm() + System.lineSeparator(); + try { + Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + itemsCompleted.put(url, urlFile); } catch (IOException e) { LOGGER.error("Error while writing to " + urlFile, e); } @@ -207,12 +208,12 @@ public abstract class AbstractJSONRipper extends AbstractRipper { /** * Cleans up & tells user about successful download */ - public void downloadCompleted(URL url, File saveAs) { + public void downloadCompleted(URL url, Path saveAs) { if (observer == null) { return; } try { - String path = Utils.removeCWD(saveAs.toPath()); + String path = Utils.removeCWD(saveAs); RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); itemsPending.remove(url); itemsCompleted.put(url, saveAs); @@ -250,7 +251,7 @@ public abstract class AbstractJSONRipper extends AbstractRipper { } itemsPending.remove(url); - itemsCompleted.put(url, file.toFile()); + itemsCompleted.put(url, file); observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); checkIfComplete(); diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index d1267adf..e31d9c0f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -452,7 +452,7 @@ public abstract class AbstractRipper * @param saveAs * Where the downloaded file is stored. */ - public abstract void downloadCompleted(URL url, File saveAs); + public abstract void downloadCompleted(URL url, Path saveAs); /** * Notifies observers that a file could not be downloaded (includes a reason). * @param url diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 5f7e4588..e0874cd2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -1,19 +1,22 @@ package com.rarchives.ripme.ripper; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.file.Path; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; +import java.io.File; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + // Should this file even exist? It does the same thing as abstractHTML ripper /**' @@ -22,7 +25,7 @@ import com.rarchives.ripme.utils.Utils; public abstract class AlbumRipper extends AbstractRipper { private Map itemsPending = Collections.synchronizedMap(new HashMap()); - private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); + private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); private Map itemsErrored = Collections.synchronizedMap(new HashMap()); protected AlbumRipper(URL url) throws IOException { @@ -68,11 +71,11 @@ public abstract class AlbumRipper extends AbstractRipper { } if (Utils.getConfigBoolean("urls_only.save", false)) { // Output URL to file - String urlFile = this.workingDir + File.separator + "urls.txt"; - try (FileWriter fw = new FileWriter(urlFile, true)) { - fw.write(url.toExternalForm()); - fw.write(System.lineSeparator()); - itemsCompleted.put(url, new File(urlFile)); + Path urlFile = Paths.get(this.workingDir + "/urls.txt"); + String text = url.toExternalForm() + System.lineSeparator(); + try { + Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + itemsCompleted.put(url, urlFile); } catch (IOException e) { LOGGER.error("Error while writing to " + urlFile, e); } @@ -114,12 +117,12 @@ public abstract class AlbumRipper extends AbstractRipper { /** * Cleans up & tells user about successful download */ - public void downloadCompleted(URL url, File saveAs) { + public void downloadCompleted(URL url, Path saveAs) { if (observer == null) { return; } try { - String path = Utils.removeCWD(saveAs.toPath()); + String path = Utils.removeCWD(saveAs); RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); itemsPending.remove(url); itemsCompleted.put(url, saveAs); @@ -157,7 +160,7 @@ public abstract class AlbumRipper extends AbstractRipper { } itemsPending.remove(url); - itemsCompleted.put(url, file.toFile()); + itemsCompleted.put(url, file); observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); checkIfComplete(); diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 82fd35d5..435523d8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -315,7 +315,7 @@ class DownloadFileThread extends Thread { return; } } while (true); - observer.downloadCompleted(url, saveAs); + observer.downloadCompleted(url, saveAs.toPath()); logger.info("[+] Saved " + url + " as " + this.prettySaveAs); } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index df518d4d..001e16ec 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -137,7 +137,7 @@ class DownloadVideoThread extends Thread { return; } } while (true); - observer.downloadCompleted(url, saveAs.toFile()); + observer.downloadCompleted(url, saveAs); logger.info("[+] Saved " + url + " as " + this.prettySaveAs); } diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 001a42d1..54e624ce 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -118,13 +118,13 @@ public abstract class VideoRipper extends AbstractRipper { * @param saveAs Path to file, including filename. */ @Override - public void downloadCompleted(URL url, File saveAs) { + public void downloadCompleted(URL url, Path saveAs) { if (observer == null) { return; } try { - String path = Utils.removeCWD(saveAs.toPath()); + String path = Utils.removeCWD(saveAs); RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); observer.update(this, msg); From fbd6feb86723a1acb08cec72c9334b3b692c0b73 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 3 Jan 2022 22:28:57 +0100 Subject: [PATCH 283/512] newgroundsrippertest flaky --- .../ripme/tst/ripper/rippers/NewgroundsRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NewgroundsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NewgroundsRipperTest.java index 4421b267..6873c82f 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NewgroundsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NewgroundsRipperTest.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.NewgroundsRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -9,6 +10,7 @@ import java.net.URL; public class NewgroundsRipperTest extends RippersTest { @Test + @Tag("flaky") public void testNewgroundsRip() throws IOException { NewgroundsRipper ripper = new NewgroundsRipper(new URL("https://zone-sama.newgrounds.com/art")); testRipper(ripper); From d4dc7a7fffa332a7fba7935ccb6dec2f40ffcd4c Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 12 Feb 2022 10:36:28 +0100 Subject: [PATCH 284/512] current way of gradle cache, use javac release flag --- .github/workflows/gradle.yml | 20 +++++--------------- build.gradle.kts | 14 +++++++++++--- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index b081b934..e9f222ae 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -30,25 +30,15 @@ jobs: id: ci-env uses: FranzDiebold/github-env-vars-action@v2 - - name: Set up Java - uses: actions/setup-java@v2 + - name: Set up java + uses: actions/setup-java@v2.5.0 with: - distribution: 'temurin' java-version: ${{ matrix.java }} - - - name: Cache Gradle packages - # speed up the build by caching dependencies, downloaded versions - uses: actions/cache@v2 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- + distribution: temurin + cache: gradle - name: Build with Gradle - run: ./gradlew build + run: gradle clean build -PjavacRelease=${{ matrix.java }} - name: SHA256 if: matrix.upload diff --git a/build.gradle.kts b/build.gradle.kts index 398019ba..e61d5c8b 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,3 +1,8 @@ +// permits to start the build setting the javac release parameter, no parameter means build for java8: +// gradle clean build -PjavacRelease=8 +// gradle clean build -PjavacRelease=17 +val javacRelease = (project.findProperty("javacRelease") ?: "8") as String + plugins { id("fr.brouillard.oss.gradle.jgitver") version "0.9.1" id("jacoco") @@ -38,9 +43,12 @@ jgitver { useGitCommitID = true } -java { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 +tasks.compileJava { + if (JavaVersion.current().isJava8) { + java.targetCompatibility = JavaVersion.VERSION_1_8 + } else { + options.release.set(Integer.parseInt(javacRelease)) + } } tasks.withType { From 8710220b893930c00109da299eb45f023bc60911 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 12 Feb 2022 10:53:00 +0100 Subject: [PATCH 285/512] thechive flaky --- .../rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java index a0093213..e7bdae85 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java @@ -63,6 +63,7 @@ public class ThechiveRipperTest extends RippersTest { * "i.thechive.com" test. */ @Test + @Tag("flaky") public void testIDotThechive() throws IOException { ThechiveRipper ripper = new ThechiveRipper(new URL("https://i.thechive.com/witcheva")); testRipper(ripper); From 0adb70d66ebf0c216baa4f35cb65738938d69d08 Mon Sep 17 00:00:00 2001 From: Zsombor Gegesy Date: Thu, 24 Feb 2022 00:34:34 +0100 Subject: [PATCH 286/512] Fix to not fail on deleted comments --- .../ripme/ripper/rippers/RedditRipper.java | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index da66b536..0a0fa306 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -311,7 +311,7 @@ public class RedditRipper extends AlbumRipper { } } - private ContainerTag getComments(JSONArray comments, String author) { + private ContainerTag getComments(JSONArray comments, String author) { ContainerTag commentsDiv = div().withId("comments"); for (int i = 0; i < comments.length(); i++) { @@ -330,24 +330,26 @@ public class RedditRipper extends AlbumRipper { return commentsDiv; } - private ContainerTag getNestedComments(JSONObject data, ContainerTag parentDiv, String author) { + private ContainerTag getNestedComments(JSONObject data, ContainerTag parentDiv, String author) { if (data.has("replies") && data.get("replies") instanceof JSONObject) { - for (int i = 0; i <= data.getJSONObject("replies").getJSONObject("data").getJSONArray("children").length() - 1; i++) { - JSONObject nestedComment = data.getJSONObject("replies") - .getJSONObject("data") - .getJSONArray("children") + JSONArray commentChildren = data.getJSONObject("replies").getJSONObject("data").getJSONArray("children"); + for (int i = 0; i < commentChildren.length(); i++) { + JSONObject nestedComment = commentChildren .getJSONObject(i).getJSONObject("data"); - ContainerTag childDiv = - div( - div( - span(nestedComment.getString("author")).withClasses("author", iff(nestedComment.getString("author").equals(author), "op")), - a(new Date((long) nestedComment.getInt("created") * 1000).toString()).withHref("#" + nestedComment.getString("name")) - ).withClass("comment").withId(nestedComment.getString("name")) - .with(rawHtml(Jsoup.parse(nestedComment.getString("body_html")).text())) - ).withClass("child"); + String nestedCommentAuthor = nestedComment.optString("author"); + if (!nestedCommentAuthor.isBlank()) { + ContainerTag childDiv = + div( + div( + span(nestedCommentAuthor).withClasses("author", iff(nestedCommentAuthor.equals(author), "op")), + a(new Date((long) nestedComment.getInt("created") * 1000).toString()).withHref("#" + nestedComment.getString("name")) + ).withClass("comment").withId(nestedComment.getString("name")) + .with(rawHtml(Jsoup.parse(nestedComment.getString("body_html")).text())) + ).withClass("child"); - parentDiv.with(getNestedComments(nestedComment, childDiv, author)); + parentDiv.with(getNestedComments(nestedComment, childDiv, author)); + } } } return parentDiv; From abb1eaa49ded5d1c4606723e687d534516d13695 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 27 Feb 2022 02:17:52 +0100 Subject: [PATCH 287/512] update dependency versions, log4j, graalvm.js, setup-java --- .github/workflows/gradle.yml | 2 +- build.gradle.kts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index e9f222ae..40347363 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -31,7 +31,7 @@ jobs: uses: FranzDiebold/github-env-vars-action@v2 - name: Set up java - uses: actions/setup-java@v2.5.0 + uses: actions/setup-java@v3.0.0 with: java-version: ${{ matrix.java }} distribution: temurin diff --git a/build.gradle.kts b/build.gradle.kts index e61d5c8b..53a736aa 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -26,9 +26,9 @@ dependencies { implementation("commons-io:commons-io:2.11.0") implementation("org.apache.httpcomponents:httpclient:4.5.13") implementation("org.apache.httpcomponents:httpmime:4.5.13") - implementation("org.apache.logging.log4j:log4j-api:2.17.0") - implementation("org.apache.logging.log4j:log4j-core:2.17.0") - implementation("org.graalvm.js:js:21.3.0") + implementation("org.apache.logging.log4j:log4j-api:2.17.1") + implementation("org.apache.logging.log4j:log4j-core:2.17.1") + implementation("org.graalvm.js:js:22.0.0.2") testImplementation(enforcedPlatform("org.junit:junit-bom:5.8.2")) testImplementation("org.junit.jupiter:junit-jupiter") } From a0619266530bd687cea90804fa1983c719ed58f3 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 28 Feb 2022 00:42:06 +0100 Subject: [PATCH 288/512] java-11 now necessary to run ripme --- .github/workflows/gradle.yml | 4 ++-- build.gradle.kts | 8 ++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 40347363..14e3b7a8 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -15,8 +15,8 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - java: [8] - include: # test newest java on one os only, upload from ubuntu java8 + java: [11] + include: # test newest java on one os only, upload from ubuntu java11 - os: ubuntu-latest java: 17 - os: ubuntu-latest diff --git a/build.gradle.kts b/build.gradle.kts index 53a736aa..9e441b35 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,7 +1,7 @@ // permits to start the build setting the javac release parameter, no parameter means build for java8: // gradle clean build -PjavacRelease=8 // gradle clean build -PjavacRelease=17 -val javacRelease = (project.findProperty("javacRelease") ?: "8") as String +val javacRelease = (project.findProperty("javacRelease") ?: "11") as String plugins { id("fr.brouillard.oss.gradle.jgitver") version "0.9.1" @@ -44,11 +44,7 @@ jgitver { } tasks.compileJava { - if (JavaVersion.current().isJava8) { - java.targetCompatibility = JavaVersion.VERSION_1_8 - } else { - options.release.set(Integer.parseInt(javacRelease)) - } + options.release.set(Integer.parseInt(javacRelease)) } tasks.withType { From c73092a3593af41c80f8f33cad8196c5b008b885 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 28 Feb 2022 00:49:58 +0100 Subject: [PATCH 289/512] xhamster tests flaky --- .../rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java index 7c5aed9f..61b35273 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java @@ -13,6 +13,7 @@ import org.junit.jupiter.api.Test; public class XhamsterRipperTest extends RippersTest { @Test + @Tag("flaky") public void testXhamsterAlbum1() throws IOException { XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/photos/gallery/sexy-preggo-girls-9026608")); testRipper(ripper); @@ -24,6 +25,7 @@ public class XhamsterRipperTest extends RippersTest { testRipper(ripper); } @Test + @Tag("flaky") public void testXhamsterAlbum2() throws IOException { XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664")); testRipper(ripper); @@ -47,6 +49,7 @@ public class XhamsterRipperTest extends RippersTest { testRipper(ripper); } @Test + @Tag("flaky") public void testBrazilianXhamster() throws IOException { XhamsterRipper ripper = new XhamsterRipper(new URL("https://pt.xhamster.com/photos/gallery/cartoon-babe-15786301")); testRipper(ripper); From 579a40d167e322ac1458fbeaa51807cb5e43e510 Mon Sep 17 00:00:00 2001 From: Edwin Date: Thu, 3 Mar 2022 00:27:07 -0500 Subject: [PATCH 290/512] Created mrcong.com ripper --- .../ripme/ripper/rippers/MrCongRipper.java | 242 ++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/MrCongRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MrCongRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MrCongRipper.java new file mode 100644 index 00000000..ab413860 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MrCongRipper.java @@ -0,0 +1,242 @@ +package com.rarchives.ripme.ripper.rippers; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.utils.Http; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + + +public class MrCongRipper extends AbstractHTMLRipper { + + private Document currDoc; + private int lastPageNum; + private int currPageNum; + private boolean tagPage = false; + + public MrCongRipper(URL url) throws IOException { + super(url); + currPageNum = 1; + } + + @Override + public String getHost() { + return "mrcong"; + } + + @Override + public String getDomain() { + return "mrcong.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + System.out.println(url.toExternalForm()); + Pattern p = Pattern.compile("^https?://mrcong\\.com/(\\S*)[0-9]+-anh(-[0-9]+-videos)?(|/|/[0-9]+)$"); + Pattern p2 = Pattern.compile("^https?://mrcong\\.com/tag/(\\S*)/$"); //Added 6-10-21 + Matcher m = p.matcher(url.toExternalForm()); + Matcher m2 = p2.matcher(url.toExternalForm()); //6-10-21 + if (m.matches()) { + return m.group(1); + } + else if(m2.matches()) { //Added 6-10-21 + tagPage = true; + System.out.println("tagPage = TRUE"); + return m2.group(1); + } + + throw new MalformedURLException("Expected mrcong.com URL format: " + + "mrcong.com/GALLERY_NAME(-anh OR -anh/ OR -anh/PAGE_NUMBER OR -anh/PAGE_NUMBER/) - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { //returns the root gallery page regardless of actual page number + // "url" is an instance field of the superclass + String rootUrlStr; + URL rootUrl; + + if(!tagPage) { + rootUrlStr = url.toExternalForm().replaceAll("(|/|/[0-9]+/?)$", "/"); + } else { //6-10-21 + rootUrlStr = url.toExternalForm().replaceAll("(page/[0-9]+/)$", "page/1/"); + } + + rootUrl = URI.create(rootUrlStr).toURL(); + url = rootUrl; + currPageNum = 1; + currDoc = Http.url(url).get(); + getMaxPageNumber(currDoc); + return currDoc; + } + + @Override + public Document getNextPage(Document doc) throws IOException { + int pageNum = currPageNum; + String urlStr; + if(!tagPage) { + if (pageNum == 1 && lastPageNum > 1) { + urlStr = url.toExternalForm().concat((pageNum + 1) + ""); + System.out.printf("Old Str: %s New Str: %s\n", url.toExternalForm(), urlStr); + } else if (pageNum < lastPageNum) { + urlStr = url.toExternalForm().replaceAll("(/([0-9]*)/?)$", ("/" + (pageNum + 1) + "/")); + System.out.printf("Old Str: %s New Str: %s\n", url.toString(), urlStr); + } else { + //System.out.printf("Error: Page number provided goes past last valid page number\n"); + throw (new IOException("Error: Page number provided goes past last valid page number\n")); + } + } else { //6-10-21 + //if (pageNum == 1 && lastPageNum >= 1) { + if (pageNum == 1 && lastPageNum > 1) { //6-10-21 + urlStr = url.toExternalForm().concat("page/" + (pageNum + 1) + ""); + System.out.printf("Old Str: %s New Str: %s\n", url.toExternalForm(), urlStr); + } else if (pageNum < lastPageNum) { + urlStr = url.toExternalForm().replaceAll("(page/([0-9]*)/?)$", ("page/" + (pageNum + 1) + "/")); + System.out.printf("Old Str: %s New Str: %s\n", url.toString(), urlStr); + } else { + //System.out.printf("Error: Page number provided goes past last valid page number\n"); + System.out.print("Error: There is no next page!\n"); + return null; + //throw (new IOException("Error: Page number provided goes past last valid page number\n")); + } + } + + url = URI.create(urlStr).toURL(); + currDoc = Http.url(url).get(); + currPageNum ++;//hi + return currDoc; + } + + private int getMaxPageNumber(Document doc) { + if(!tagPage) { + try { + lastPageNum = Integer.parseInt(doc.select("div.page-link > a").last().text()); //gets the last possible page for the gallery + } catch(Exception e) { + return 1; + } + } else { + try { + lastPageNum = Integer.parseInt(doc.select("div.pagination > a").last().text()); //gets the last possible page for the gallery + System.out.println("The last page found for " + url + " was " + lastPageNum); + } catch(Exception e) { + return 1; + } + } + + return lastPageNum; + } + + private int getCurrentPageNum(Document doc) { + int currPage; //6-10-21 + + if(!tagPage) { + currPage = Integer.parseInt(doc.select("div.page-link > span").first().text()); + } else { + currPage = Integer.parseInt(doc.select("div.pagination > span").first().text()); + } + + System.out.println("The current page was found to be: " + currPage); + + return currPage; + } + + @Override + public List getURLsFromPage(Document doc) { //gets the urls of the images + List result = new ArrayList<>(); + + if(!tagPage) { + for (Element el : doc.select("p > img")) { + String imageSource = el.attr("src"); + result.add(imageSource); + } + + System.out.println("\n1.)Printing List: " + result + "\n"); + } else { //6-10-21 + //List gallery_set_list = new ArrayList<>(); + + for (Element el : doc.select("h2 > a")) { + String pageSource = el.attr("href"); + if(!pageSource.equals("https://mrcong.com/")) { + result.add(pageSource); + System.out.println("\n" + pageSource + " has been added to the list."); + } + } + + /*for (String el2 : gallery_set_list) { + try { + URL temp_urL = URI.create(el2).toURL(); + MrCongRipper mcr = new MrCongRipper(temp_urL); + System.out.println("URL being ripped: " + mcr.url.toString()); + result.addAll(mcr.getURLsFromPage(mcr.getFirstPage())); + + Document nextPg = mcr.getNextPage(mcr.currDoc); + while(nextPg != null) { + result.addAll(mcr.getURLsFromPage(nextPg)); + nextPg = mcr.getNextPage(mcr.currDoc); + } + } catch (IOException e) { + e.printStackTrace(); + } + + }*/ + + System.out.println("\n2.)Printing List: " + result + "\n"); + } + + return result; + } + + @Override + public void downloadURL(URL url, int index) { + //addURLToDownload(url, getPrefix(index)); + + if(!tagPage) { + addURLToDownload(url, getPrefix(index)); + } else { + try { + List ls = this.getURLsFromPage(this.currDoc); + Document np = this.getNextPage(this.currDoc); + + while(np != null) { //Creates a list of all sets to download + ls.addAll(this.getURLsFromPage(np)); + np = this.getNextPage(np); + } + + for(String urlStr : ls) { + MrCongRipper mcr = new MrCongRipper(URI.create(urlStr).toURL()); + mcr.setup(); + mcr.rip(); + } + + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + /* + public static void main(String[] args) { + try { + MrCongRipper a = new MrCongRipper(URI.create("https://mrcong.com/ruisg-vol-084-xiao-hui-49-anh/").toURL()); + a.getFirstPage(); + System.out.println("Current Page#: " + a.getCurrentPageNum(a.currDoc)); + System.out.println("Max Page#: " + a.getMaxPageNumber(a.currDoc)); + + while(a.getNextPage() != null) { + + } + }catch(Exception IOException) { + System.out.println("IOException"); + } + } + */ +} +// https://mrcong.com/tag/zhu-ke-er/ +// https://mrcong.com/tag/%e9%9b%aa%e7%90%aasama/ \ No newline at end of file From 0ba0fc8cc7f44484a87f550e4035d505c34c15e9 Mon Sep 17 00:00:00 2001 From: Edwin Date: Thu, 3 Mar 2022 00:27:35 -0500 Subject: [PATCH 291/512] Created mrcong.com ripper --- .../ripme/ripper/rippers/MrCongRipper.java | 21 +------------------ 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MrCongRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MrCongRipper.java index ab413860..209db9cf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MrCongRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MrCongRipper.java @@ -220,23 +220,4 @@ public class MrCongRipper extends AbstractHTMLRipper { } } } - - /* - public static void main(String[] args) { - try { - MrCongRipper a = new MrCongRipper(URI.create("https://mrcong.com/ruisg-vol-084-xiao-hui-49-anh/").toURL()); - a.getFirstPage(); - System.out.println("Current Page#: " + a.getCurrentPageNum(a.currDoc)); - System.out.println("Max Page#: " + a.getMaxPageNumber(a.currDoc)); - - while(a.getNextPage() != null) { - - } - }catch(Exception IOException) { - System.out.println("IOException"); - } - } - */ -} -// https://mrcong.com/tag/zhu-ke-er/ -// https://mrcong.com/tag/%e9%9b%aa%e7%90%aasama/ \ No newline at end of file +} \ No newline at end of file From 96acdb0fe3fd2d7200e139cda411bcbe449d4427 Mon Sep 17 00:00:00 2001 From: Zsombor Gegesy Date: Thu, 3 Mar 2022 21:07:39 +0100 Subject: [PATCH 292/512] Set workingDir even if the file exists - this prevents an NPE later --- .../java/com/rarchives/ripme/ripper/AbstractJSONRipper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 021e5c64..47a13717 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -293,9 +293,9 @@ public abstract class AbstractJSONRipper extends AbstractRipper { if (!Files.exists(wd)) { LOGGER.info("[+] Creating directory: " + Utils.removeCWD(wd)); Files.createDirectory(wd); - this.workingDir = wd.toFile(); } - LOGGER.debug("Set working directory to: " + this.workingDir); + this.workingDir = wd.toFile(); + LOGGER.info("Set working directory to: {}", this.workingDir); } /** From b4ee50d5d1096ff7e6ca2d2835527e6224978425 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 11 Mar 2022 04:55:11 +0100 Subject: [PATCH 293/512] MyhentaicomicsRipperTest flaky --- .../ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java index 46256168..798176d9 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java @@ -5,10 +5,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.MyhentaicomicsRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class MyhentaicomicsRipperTest extends RippersTest { @Test + @Tag("flaky") public void testMyhentaicomicsAlbum() throws IOException { MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(new URL("http://myhentaicomics.com/index.php/Nienna-Lost-Tales")); testRipper(ripper); @@ -25,6 +27,7 @@ public class MyhentaicomicsRipperTest extends RippersTest { Assertions.assertEquals("2409", ripper.getGID(new URL("http://myhentaicomics.com/index.php/tag/2409/"))); } @Test + @Tag("flaky") public void testGetAlbumsToQueue() throws IOException { URL url = new URL("https://myhentaicomics.com/index.php/tag/3167/"); MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(url); From 536339dd789c4d6dc4401dce37202b1f6a2a1a29 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 11 Mar 2022 05:49:53 +0100 Subject: [PATCH 294/512] workingdir exists, print error if not, but continue. fixes #74 --- .../java/com/rarchives/ripme/ui/MainWindow.java | 14 ++------------ src/main/java/com/rarchives/ripme/utils/Utils.java | 9 +++++++-- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 884df89f..96208632 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -880,12 +880,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { }); configSaveDirButton.addActionListener(arg0 -> { UIManager.put("FileChooser.useSystemExtensionHiding", false); - JFileChooser jfc = null; - try { - jfc = new JFileChooser(Utils.getWorkingDirectory().toString()); - } catch (IOException e) { - e.printStackTrace(); - } + JFileChooser jfc = new JFileChooser(Utils.getWorkingDirectory().toString()); jfc.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); int returnVal = jfc.showDialog(null, "select directory"); if (returnVal != JFileChooser.APPROVE_OPTION) { @@ -904,12 +899,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { }); configUrlFileChooserButton.addActionListener(arg0 -> { UIManager.put("FileChooser.useSystemExtensionHiding", false); - JFileChooser jfc = null; - try { - jfc = new JFileChooser(Utils.getWorkingDirectory().toAbsolutePath().toString()); - } catch (IOException e) { - e.printStackTrace(); - } + JFileChooser jfc = new JFileChooser(Utils.getWorkingDirectory().toAbsolutePath().toString()); jfc.setFileSelectionMode(JFileChooser.FILES_ONLY); int returnVal = jfc.showDialog(null, "Open"); if (returnVal != JFileChooser.APPROVE_OPTION) { diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 7a230ed1..825d5142 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -109,7 +109,7 @@ public class Utils { * * @return Root directory to save rips to. */ - public static Path getWorkingDirectory() throws IOException { + public static Path getWorkingDirectory() { String currentDir = getJarDirectory() + File.separator + RIP_DIRECTORY + File.separator; if (config != null) { @@ -118,7 +118,12 @@ public class Utils { Path workingDir = Paths.get(currentDir); if (!Files.exists(workingDir)) { - Files.createDirectory(workingDir); + try { + Files.createDirectory(workingDir); + } catch (IOException e) { + LOGGER.error("WorkingDir " + workingDir + " not exists, and could not be created. Set to user.home, continue."); + workingDir = Paths.get(System.getProperty("user.home")); + } } return workingDir; } From 774805892de4d444b09269e1412c41e3b0097043 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 1 Apr 2022 06:34:31 +0200 Subject: [PATCH 295/512] update gradle-7.3.3 --> gradle 7.4.2 --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 2e6e5897..aa991fce 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 76b2bc80bbe5981b53dae26a62eb87db4aefa948 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 1 Apr 2022 06:43:57 +0200 Subject: [PATCH 296/512] textGetNextPage Soundgasm test flaky --- .../ripme/tst/ripper/rippers/SoundgasmRipperTest.java | 3 +++ .../rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java | 1 + 2 files changed, 4 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java index 76d7bd20..8a19fa12 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.RedditRipper; import com.rarchives.ripme.ripper.rippers.SoundgasmRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -10,12 +11,14 @@ import java.net.URL; public class SoundgasmRipperTest extends RippersTest { @Test + @Tag("flaky") public void testSoundgasmURLs() throws IOException { SoundgasmRipper ripper = new SoundgasmRipper(new URL("https://soundgasm.net/u/_Firefly_xoxo/Rambles-with-my-Lovense")); testRipper(ripper); } @Test + @Tag("flaky") public void testRedditSoundgasmURL() throws IOException { RedditRipper ripper = new RedditRipper(new URL("https://www.reddit.com/r/gonewildaudio/comments/kn1bvj/f4m_mistress_controlled_my_lovense_while_i_tried/")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java index 61b35273..1f1af488 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java @@ -61,6 +61,7 @@ public class XhamsterRipperTest extends RippersTest { Assertions.assertEquals("7254664", ripper.getGID(url)); } @Test + @Tag("flaky") public void testGetNextPage() throws IOException { XhamsterRipper ripper = new XhamsterRipper(new URL("https://pt.xhamster.com/photos/gallery/mega-compil-6-10728626")); Document doc = ripper.getFirstPage(); From 9d66f41584fefc479d5bcba947ecae8ecb65863e Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 2 Apr 2022 09:32:58 +0200 Subject: [PATCH 297/512] remove method removeCWD(String) --- .../com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 2 +- .../com/rarchives/ripme/ripper/AbstractRipper.java | 2 +- src/main/java/com/rarchives/ripme/utils/Utils.java | 10 ---------- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 951562e5..861410b7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -265,7 +265,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { } LOGGER.debug("Downloading " + url + "'s description to " + saveFileAs); if (!saveFileAs.getParentFile().exists()) { - LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); + LOGGER.info("[+] Creating directory: " + saveFileAs.getParent()); saveFileAs.getParentFile().mkdirs(); } return true; diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index e31d9c0f..407aa33a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -339,7 +339,7 @@ public abstract class AbstractRipper } LOGGER.debug("Downloading " + url + " to " + saveFileAs); if (!saveFileAs.getParentFile().exists()) { - LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); + LOGGER.info("[+] Creating directory: " + saveFileAs.getParent()); saveFileAs.getParentFile().mkdirs(); } if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) { diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 825d5142..7323759e 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -362,16 +362,6 @@ public class Utils { return url; } - /** - * Removes the current working directory from a given filename - * - * @param file Path to the file - * @return 'file' without the leading current working directory - */ - public static String removeCWD(String file) { - return removeCWD(Paths.get(file)); - } - /** * Get a list of all Classes within a package. Works with file system projects * and jar files! Borrowed from StackOverflow, but I don't have a link :[ From 5c9c13abcd709cc4936728f94b10fb49882c9087 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 2 Apr 2022 08:33:06 +0200 Subject: [PATCH 298/512] remove warnings in AbstractRipper AbstractHTMLRipper DownloadFileThread Utils --- .../ripme/ripper/AbstractHTMLRipper.java | 16 ++---- .../ripme/ripper/AbstractRipper.java | 10 +--- .../ripme/ripper/DownloadFileThread.java | 46 ++++++----------- .../java/com/rarchives/ripme/utils/Utils.java | 51 +++++++------------ 4 files changed, 40 insertions(+), 83 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 861410b7..45378bfa 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -309,7 +309,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { */ public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { // Only download one file if this is a test. - if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { + if (isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { stop(); itemsPending.clear(); return false; @@ -388,7 +388,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { } @Override - /** + /* * Cleans up & tells user about failed download. */ public void downloadErrored(URL url, String reason) { @@ -436,8 +436,6 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { * Sets directory to save all ripped files to. * @param url * URL to define how the working directory should be saved. - * @throws - * IOException */ @Override public void setWorkingDir(URL url) throws IOException { @@ -447,12 +445,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { if (!path.endsWith(File.separator)) { path += File.separator; } - String title; - if (Utils.getConfigBoolean("album_titles.save", true)) { - title = getAlbumTitle(this.url); - } else { - title = super.getAlbumTitle(this.url); - } + String title = getAlbumTitle(this.url); LOGGER.debug("Using album title '" + title + "'"); title = Utils.filesystemSafe(title); @@ -485,12 +478,11 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { */ @Override public String getStatusText() { - String sb = getCompletionPercentage() + + return getCompletionPercentage() + "% " + "- Pending: " + itemsPending.size() + ", Completed: " + itemsCompleted.size() + ", Errored: " + itemsErrored.size(); - return sb; } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 407aa33a..1384b8f1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -243,7 +243,7 @@ public abstract class AbstractRipper */ protected boolean addURLToDownload(URL url, Map options, Map cookies) { // Bit of a hack but this lets us pass a bool using a map - boolean useMIME = options.getOrDefault("getFileExtFromMIME", "false").toLowerCase().equals("true"); + boolean useMIME = options.getOrDefault("getFileExtFromMIME", "false").equalsIgnoreCase("true"); return addURLToDownload(url, options.getOrDefault("prefix", ""), options.getOrDefault("subdirectory", ""), options.getOrDefault("referrer", null), cookies, options.getOrDefault("fileName", null), options.getOrDefault("extension", null), useMIME); } @@ -455,15 +455,11 @@ public abstract class AbstractRipper public abstract void downloadCompleted(URL url, Path saveAs); /** * Notifies observers that a file could not be downloaded (includes a reason). - * @param url - * @param reason */ public abstract void downloadErrored(URL url, String reason); /** * Notify observers that a download could not be completed, * but was not technically an "error". - * @param url - * @param file */ public abstract void downloadExists(URL url, Path file); @@ -581,7 +577,6 @@ public abstract class AbstractRipper * The package name. * @return * List of constructors for all eligible Rippers. - * @throws Exception */ public static List> getRipperConstructors(String pkg) throws Exception { List> constructors = new ArrayList<>(); @@ -595,8 +590,7 @@ public abstract class AbstractRipper /** * Sends an update message to the relevant observer(s) on this ripper. - * @param status - * @param message + * @param status */ public void sendUpdate(STATUS status, Object message) { if (observer == null) { diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 435523d8..562ac366 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -10,11 +10,9 @@ import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; import java.util.Map; -import java.util.ResourceBundle; import javax.net.ssl.HttpsURLConnection; -import com.rarchives.ripme.ui.MainWindow; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jsoup.HttpStatusException; @@ -32,12 +30,12 @@ class DownloadFileThread extends Thread { private String referrer = ""; private Map cookies = new HashMap<>(); - private URL url; + private final URL url; private File saveAs; - private String prettySaveAs; - private AbstractRipper observer; - private int retries; - private Boolean getFileExtFromMIME; + private final String prettySaveAs; + private final AbstractRipper observer; + private final int retries; + private final Boolean getFileExtFromMIME; private final int TIMEOUT; @@ -69,7 +67,7 @@ class DownloadFileThread extends Thread { saveAs = new File( saveAs.getParentFile().getAbsolutePath() + File.separator + Utils.sanitizeSaveAs(saveAs.getName())); long fileSize = 0; - int bytesTotal = 0; + int bytesTotal; int bytesDownloaded = 0; if (saveAs.exists() && observer.tryResumeDownload()) { fileSize = saveAs.length(); @@ -85,7 +83,7 @@ class DownloadFileThread extends Thread { && !observer.tryResumeDownload()) { if (Utils.getConfigBoolean("file.overwrite", false)) { logger.info("[!] " + Utils.getLocalizedString("deleting.existing.file") + prettySaveAs); - saveAs.delete(); + if (!saveAs.delete()) logger.error("could not delete existing file: " + saveAs.getAbsolutePath()); } else { logger.info("[!] " + Utils.getLocalizedString("skipping") + " " + url + " -- " + Utils.getLocalizedString("file.already.exists") + ": " + prettySaveAs); @@ -98,8 +96,6 @@ class DownloadFileThread extends Thread { int tries = 0; // Number of attempts to download do { tries += 1; - InputStream bis = null; - OutputStream fos = null; try { logger.info(" Downloading file: " + urlToDownload + (tries > 0 ? " Retry #" + tries : "")); observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); @@ -122,14 +118,14 @@ class DownloadFileThread extends Thread { huc.setRequestProperty("Referer", referrer); // Sic } huc.setRequestProperty("User-agent", AbstractRipper.USER_AGENT); - String cookie = ""; + StringBuilder cookie = new StringBuilder(); for (String key : cookies.keySet()) { - if (!cookie.equals("")) { - cookie += "; "; + if (!cookie.toString().equals("")) { + cookie.append("; "); } - cookie += key + "=" + cookies.get(key); + cookie.append(key).append("=").append(cookies.get(key)); } - huc.setRequestProperty("Cookie", cookie); + huc.setRequestProperty("Cookie", cookie.toString()); if (observer.tryResumeDownload()) { if (fileSize != 0) { huc.setRequestProperty("Range", "bytes=" + fileSize + "-"); @@ -187,6 +183,7 @@ class DownloadFileThread extends Thread { } // Save file + InputStream bis; bis = new BufferedInputStream(huc.getInputStream()); // Check if we should get the file ext from the MIME type @@ -212,6 +209,7 @@ class DownloadFileThread extends Thread { } } // If we're resuming a download we append data to the existing file + OutputStream fos = null; if (statusCode == 206) { fos = new FileOutputStream(saveAs, true); } else { @@ -240,7 +238,9 @@ class DownloadFileThread extends Thread { // not allow fos = Files.newOutputStream( Utils.shortenSaveAsWindows(saveAs.getParentFile().getPath(), saveAs.getName())); + assert fos != null: "After shortenSaveAsWindows: " + saveAs.getAbsolutePath(); } + assert fos != null: e.getStackTrace(); } } byte[] data = new byte[1024 * 256]; @@ -292,20 +292,6 @@ class DownloadFileThread extends Thread { Utils.getLocalizedString("failed.to.download") + " " + url.toExternalForm()); return; - }finally { - // Close any open streams - try { - if (bis != null) { - bis.close(); - } - } catch (IOException e) { - } - try { - if (fos != null) { - fos.close(); - } - } catch (IOException e) { - } } if (tries > this.retries) { logger.error("[!] " + Utils.getLocalizedString("exceeded.maximum.retries") + " (" + this.retries diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 7323759e..f3b20a5f 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -20,13 +20,13 @@ import javax.sound.sampled.LineEvent; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.lang.reflect.Constructor; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLDecoder; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.FileSystems; import java.nio.file.Files; @@ -62,7 +62,7 @@ public class Utils { private static final int SHORTENED_PATH_LENGTH = 12; private static PropertiesConfiguration config; - private static HashMap> cookieCache; + private static final HashMap> cookieCache; private static final HashMap magicHash = new HashMap<>(); private static ResourceBundle resourceBundle; @@ -256,11 +256,7 @@ public class Utils { */ private static boolean portableMode() { Path file = getJarDirectory().resolve(CONFIG_FILE); - if (Files.exists(file) && !Files.isDirectory(file)) { - return true; - } - - return false; + return Files.exists(file) && !Files.isDirectory(file); } /** @@ -326,8 +322,7 @@ public class Utils { * @return saveAs in relation to the CWD */ public static String removeCWD(Path saveAs) { - String prettySaveAs = saveAs.relativize(Paths.get(".").toAbsolutePath()).toString(); - return prettySaveAs; + return saveAs.relativize(Paths.get(".").toAbsolutePath()).toString(); } /** @@ -409,7 +404,7 @@ public class Utils { // Load from JAR try { String jarPath = fullPath.replaceFirst("[.]jar[!].*", ".jar").replaceFirst("file:", ""); - jarPath = URLDecoder.decode(jarPath, "UTF-8"); + jarPath = URLDecoder.decode(jarPath, StandardCharsets.UTF_8); JarFile jarFile = new JarFile(jarPath); Enumeration entries = jarFile.entries(); while (entries.hasMoreElements()) { @@ -662,18 +657,13 @@ public class Utils { String[] parts = query.split("&"); int pos; - try { - for (String part : parts) { - if ((pos = part.indexOf('=')) >= 0) { - res.put(URLDecoder.decode(part.substring(0, pos), "UTF-8"), - URLDecoder.decode(part.substring(pos + 1), "UTF-8")); - } else { - res.put(URLDecoder.decode(part, "UTF-8"), ""); - } + for (String part : parts) { + if ((pos = part.indexOf('=')) >= 0) { + res.put(URLDecoder.decode(part.substring(0, pos), StandardCharsets.UTF_8), + URLDecoder.decode(part.substring(pos + 1), StandardCharsets.UTF_8)); + } else { + res.put(URLDecoder.decode(part, StandardCharsets.UTF_8), ""); } - } catch (UnsupportedEncodingException e) { - // Shouldn't happen since UTF-8 is required to be supported - throw new RuntimeException(e); } return res; @@ -694,20 +684,15 @@ public class Utils { String[] parts = query.split("&"); int pos; - try { - for (String part : parts) { - if ((pos = part.indexOf('=')) >= 0) { - if (URLDecoder.decode(part.substring(0, pos), "UTF-8").equals(key)) { - return URLDecoder.decode(part.substring(pos + 1), "UTF-8"); - } - - } else if (URLDecoder.decode(part, "UTF-8").equals(key)) { - return ""; + for (String part : parts) { + if ((pos = part.indexOf('=')) >= 0) { + if (URLDecoder.decode(part.substring(0, pos), StandardCharsets.UTF_8).equals(key)) { + return URLDecoder.decode(part.substring(pos + 1), StandardCharsets.UTF_8); } + + } else if (URLDecoder.decode(part, StandardCharsets.UTF_8).equals(key)) { + return ""; } - } catch (UnsupportedEncodingException e) { - // Shouldn't happen since UTF-8 is required to be supported - throw new RuntimeException(e); } return null; From 4f2a5095bd592cce7a7e48474066b45152230860 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 10 Apr 2022 13:20:54 +0200 Subject: [PATCH 299/512] try getData(DataFlavor.getTextPlainUnicodeFlavor()) --- src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java b/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java index 24c46cd4..c149d6fe 100644 --- a/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java @@ -34,7 +34,7 @@ class ClipboardUtils { return (String) Toolkit .getDefaultToolkit() .getSystemClipboard() - .getData(DataFlavor.stringFlavor); + .getData(DataFlavor.getTextPlainUnicodeFlavor()); } catch (IllegalStateException e) { e.printStackTrace(); logger.error("Caught and recovered from IllegalStateException: " + e.getMessage()); From f355b639c6d97d701b71be0ad00c44c11e898a57 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 15 Apr 2022 11:06:42 +0200 Subject: [PATCH 300/512] remove .vscode, debug works automatically in recent vscode open one of the java files, and do "run - start debugging" will launch the app for debugging. --- .vscode/settings.json | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index e26479b6..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "files.exclude": { - "target/**": true, - "**/.git": true, - "**/.DS_Store": true, - "**/*.class": true, - "**/rips/**": true - }, - "java.configuration.updateBuildConfiguration": "automatic" -} From e081e927f0fceabbe971ff70f73f4cffecddebe6 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 15 Apr 2022 11:19:13 +0200 Subject: [PATCH 301/512] java-11 now, mention in readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 952b91fa..e4afcabe 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ RipMe is maintained with ♥️ and in our limited free time by **[@MetaPrime](h # About -RipMe is an album ripper for various websites. It is a cross-platform tool that runs on your computer, and requires Java 8. RipMe has been tested and confirmed working on Windows, Linux and MacOS. +RipMe is an album ripper for various websites. It is a cross-platform tool that runs on your computer, and requires Java 11. RipMe has been tested and confirmed working on Windows, Linux and MacOS. ![Screenshot](https://i.imgur.com/UCQNjeg.png) @@ -26,7 +26,7 @@ Download `ripme.jar` from the [latest release](/releases). For information about [the How To Run wiki](https://github.com/ripmeapp/ripme/wiki/How-To-Run-RipMe). The version number like ripme-1.7.94-17-2167aa34-feature_auto_release.jar contains a release number (1.7.94), given by -a person the number of commits since this version (17). The commit SHA (2167aa34) is there uniquely referencing the +a person, the number of commits since this version (17). The commit SHA (2167aa34) uniquely references the source code ripme was built from. If it is not built from the main branch, the branch name (feature/auto-release) is given. From 6659e062709befb3c4d2f7dee0cd179ba4d8b53c Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 15 Apr 2022 13:38:05 +0200 Subject: [PATCH 302/512] sanitize filename string before converting to java Path --- src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 1384b8f1..89344930 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -419,7 +419,7 @@ public abstract class AbstractRipper if (extension != null) { saveAs = saveAs + "." + extension; } - return saveAs; + return Utils.sanitizeSaveAs(saveAs); } From 9e099de63605d32346db57a3e3a2225204c0174a Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 15 Apr 2022 14:12:12 +0200 Subject: [PATCH 303/512] reorder prefix parameter to be close to filename --- .../com/rarchives/ripme/ripper/AbstractRipper.java | 11 ++++++----- .../ripme/ripper/rippers/ComicextraRipper.java | 2 +- .../ripme/ripper/rippers/EightmusesRipper.java | 2 +- .../rarchives/ripme/ripper/rippers/TsuminoRipper.java | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 89344930..e97c18fd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -244,8 +244,9 @@ public abstract class AbstractRipper protected boolean addURLToDownload(URL url, Map options, Map cookies) { // Bit of a hack but this lets us pass a bool using a map boolean useMIME = options.getOrDefault("getFileExtFromMIME", "false").equalsIgnoreCase("true"); - return addURLToDownload(url, options.getOrDefault("prefix", ""), options.getOrDefault("subdirectory", ""), options.getOrDefault("referrer", null), - cookies, options.getOrDefault("fileName", null), options.getOrDefault("extension", null), useMIME); + return addURLToDownload(url, options.getOrDefault("subdirectory", ""), options.getOrDefault("referrer", null), cookies, + options.getOrDefault("prefix", ""), options.getOrDefault("fileName", null), options.getOrDefault("extension", null), + useMIME); } @@ -283,7 +284,7 @@ public abstract class AbstractRipper * True if downloaded successfully * False if failed to download */ - protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName, String extension, Boolean getFileExtFromMIME) { + protected boolean addURLToDownload(URL url, String subdirectory, String referrer, Map cookies, String prefix, String fileName, String extension, Boolean getFileExtFromMIME) { // A common bug is rippers adding urls that are just "http:". This rejects said urls if (url.toExternalForm().equals("http:") || url.toExternalForm().equals("https:")) { LOGGER.info(url.toExternalForm() + " is a invalid url amd will be changed"); @@ -314,7 +315,7 @@ public abstract class AbstractRipper LOGGER.debug("Ripper has been stopped"); return false; } - LOGGER.debug("url: " + url + ", prefix: " + prefix + ", subdirectory" + subdirectory + ", referrer: " + referrer + ", cookies: " + cookies + ", fileName: " + fileName); + LOGGER.debug("url: " + url + ", subdirectory" + subdirectory + ", referrer: " + referrer + ", cookies: " + cookies + ", prefix: " + prefix + ", fileName: " + fileName); String saveAs = getFileName(url, fileName, extension); File saveFileAs; try { @@ -354,7 +355,7 @@ public abstract class AbstractRipper } protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName, String extension) { - return addURLToDownload(url, prefix, subdirectory, referrer, cookies, fileName, extension, false); + return addURLToDownload(url, subdirectory, referrer, cookies, prefix, fileName, extension, false); } protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ComicextraRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ComicextraRipper.java index 08b27a76..a2e79bc0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ComicextraRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ComicextraRipper.java @@ -127,7 +127,7 @@ public class ComicextraRipper extends AbstractHTMLRipper { String subdirectory = getSubDirectoryName(); String prefix = getPrefix(++imageIndex); - addURLToDownload(url, prefix, subdirectory, null, null, FILE_NAME, null, Boolean.TRUE); + addURLToDownload(url, subdirectory, null, null, prefix, FILE_NAME, null, Boolean.TRUE); } /* diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java index 82150b89..26808aa0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java @@ -124,7 +124,7 @@ public class EightmusesRipper extends AbstractHTMLRipper { for (int i = 0; i != json.getJSONArray("pictures").length(); i++) { image = "https://www.8muses.com/image/fl/" + json.getJSONArray("pictures").getJSONObject(i).getString("publicUri"); URL imageUrl = new URL(image); - addURLToDownload(imageUrl, getPrefixShort(x), getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, "", null, true); + addURLToDownload(imageUrl, getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, getPrefixShort(x), "", null, true); // X is our page index x++; if (isThisATest()) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java index 846c4795..31917199 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java @@ -127,6 +127,6 @@ public class TsuminoRipper extends AbstractHTMLRipper { There is no way to tell if an image returned from tsumino.com is a png to jpg. The content-type header is always "image/jpeg" even when the image is a png. The file ext is not included in the url. */ - addURLToDownload(url, getPrefix(index), "", null, null, null, null, true); + addURLToDownload(url, "", null, null, getPrefix(index), null, null, true); } } From dec495f0edeca99d32e32e73d99a58f2b3f5fde6 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 15 Apr 2022 15:45:01 +0200 Subject: [PATCH 304/512] no extra variable saveAs when finding filename --- .../ripme/ripper/AbstractRipper.java | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index e97c18fd..a12f5fb2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -395,12 +395,9 @@ public abstract class AbstractRipper } public static String getFileName(URL url, String fileName, String extension) { - String saveAs; - if (fileName != null) { - saveAs = fileName; - } else { - saveAs = url.toExternalForm(); - saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1); + if (fileName == null) { + fileName = url.toExternalForm(); + fileName = fileName.substring(fileName.lastIndexOf('/')+1); } if (extension == null) { // Get the extension of the file @@ -409,18 +406,18 @@ public abstract class AbstractRipper String[] lastBit = lastBitOfURL[lastBitOfURL.length - 1].split("."); if (lastBit.length != 0) { extension = lastBit[lastBit.length - 1]; - saveAs = saveAs + "." + extension; + fileName = fileName + "." + extension; } } - if (saveAs.indexOf('?') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('?')); } - if (saveAs.indexOf('#') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('#')); } - if (saveAs.indexOf('&') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('&')); } - if (saveAs.indexOf(':') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf(':')); } + if (fileName.indexOf('?') >= 0) { fileName = fileName.substring(0, fileName.indexOf('?')); } + if (fileName.indexOf('#') >= 0) { fileName = fileName.substring(0, fileName.indexOf('#')); } + if (fileName.indexOf('&') >= 0) { fileName = fileName.substring(0, fileName.indexOf('&')); } + if (fileName.indexOf(':') >= 0) { fileName = fileName.substring(0, fileName.indexOf(':')); } if (extension != null) { - saveAs = saveAs + "." + extension; + fileName = fileName + "." + extension; } - return Utils.sanitizeSaveAs(saveAs); + return Utils.sanitizeSaveAs(fileName); } From 3b33a69bd7077718e00ffe6004e9e9bb731d7526 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 15 Apr 2022 15:51:30 +0200 Subject: [PATCH 305/512] check if empty for filename and extension, not double add extension --- src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index a12f5fb2..a2956ead 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -395,18 +395,17 @@ public abstract class AbstractRipper } public static String getFileName(URL url, String fileName, String extension) { - if (fileName == null) { + if (fileName == null || fileName.trim().isEmpty()) { fileName = url.toExternalForm(); fileName = fileName.substring(fileName.lastIndexOf('/')+1); } - if (extension == null) { + if (extension == null || extension.trim().isEmpty()) { // Get the extension of the file String[] lastBitOfURL = url.toExternalForm().split("/"); String[] lastBit = lastBitOfURL[lastBitOfURL.length - 1].split("."); if (lastBit.length != 0) { extension = lastBit[lastBit.length - 1]; - fileName = fileName + "." + extension; } } From 4be9ebaec7191188fd23fecf533526c244d402e8 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 15 Apr 2022 15:54:12 +0200 Subject: [PATCH 306/512] reorder and comment code to find filename --- .../com/rarchives/ripme/ripper/AbstractRipper.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index a2956ead..c4d046f9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -395,10 +395,17 @@ public abstract class AbstractRipper } public static String getFileName(URL url, String fileName, String extension) { + // retrieve filename from URL if not passed if (fileName == null || fileName.trim().isEmpty()) { fileName = url.toExternalForm(); fileName = fileName.substring(fileName.lastIndexOf('/')+1); } + if (fileName.indexOf('?') >= 0) { fileName = fileName.substring(0, fileName.indexOf('?')); } + if (fileName.indexOf('#') >= 0) { fileName = fileName.substring(0, fileName.indexOf('#')); } + if (fileName.indexOf('&') >= 0) { fileName = fileName.substring(0, fileName.indexOf('&')); } + if (fileName.indexOf(':') >= 0) { fileName = fileName.substring(0, fileName.indexOf(':')); } + + // retrieve extension from URL if not passed, no extension if nothing found if (extension == null || extension.trim().isEmpty()) { // Get the extension of the file String[] lastBitOfURL = url.toExternalForm().split("/"); @@ -408,14 +415,11 @@ public abstract class AbstractRipper extension = lastBit[lastBit.length - 1]; } } - - if (fileName.indexOf('?') >= 0) { fileName = fileName.substring(0, fileName.indexOf('?')); } - if (fileName.indexOf('#') >= 0) { fileName = fileName.substring(0, fileName.indexOf('#')); } - if (fileName.indexOf('&') >= 0) { fileName = fileName.substring(0, fileName.indexOf('&')); } - if (fileName.indexOf(':') >= 0) { fileName = fileName.substring(0, fileName.indexOf(':')); } + // if extension is passed or found, add it if (extension != null) { fileName = fileName + "." + extension; } + // make sure filename is not too long and has no unsupported chars return Utils.sanitizeSaveAs(fileName); } From 3eca9fa620dd1525abeaaf701708737b16a4019e Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 15 Apr 2022 16:01:03 +0200 Subject: [PATCH 307/512] assure filname with prefix is file system safe --- .../com/rarchives/ripme/ripper/AbstractRipper.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index c4d046f9..ea713789 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -10,6 +10,7 @@ import java.lang.reflect.Constructor; import java.net.MalformedURLException; import java.net.URL; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -316,14 +317,13 @@ public abstract class AbstractRipper return false; } LOGGER.debug("url: " + url + ", subdirectory" + subdirectory + ", referrer: " + referrer + ", cookies: " + cookies + ", prefix: " + prefix + ", fileName: " + fileName); - String saveAs = getFileName(url, fileName, extension); + String saveAs = getFileName(url, prefix, fileName, extension); File saveFileAs; try { if (!subdirectory.equals("")) { subdirectory = Utils.filesystemSafe(subdirectory); subdirectory = File.separator + subdirectory; } - prefix = Utils.filesystemSanitized(prefix); String topFolderName = workingDir.getCanonicalPath(); if (App.stringToAppendToFoldername != null) { topFolderName = topFolderName + App.stringToAppendToFoldername; @@ -332,7 +332,6 @@ public abstract class AbstractRipper topFolderName + subdirectory + File.separator - + prefix + saveAs); } catch (IOException e) { LOGGER.error("[!] Error creating save file path for URL '" + url + "':", e); @@ -394,7 +393,7 @@ public abstract class AbstractRipper return addURLToDownload(url, prefix, ""); } - public static String getFileName(URL url, String fileName, String extension) { + public static String getFileName(URL url, String prefix, String fileName, String extension) { // retrieve filename from URL if not passed if (fileName == null || fileName.trim().isEmpty()) { fileName = url.toExternalForm(); @@ -405,6 +404,11 @@ public abstract class AbstractRipper if (fileName.indexOf('&') >= 0) { fileName = fileName.substring(0, fileName.indexOf('&')); } if (fileName.indexOf(':') >= 0) { fileName = fileName.substring(0, fileName.indexOf(':')); } + // add prefix + if (prefix != null && !prefix.trim().isEmpty()) { + fileName = prefix + fileName; + } + // retrieve extension from URL if not passed, no extension if nothing found if (extension == null || extension.trim().isEmpty()) { // Get the extension of the file From 015d35706c0fd2e943c2f703e3b5dcecc4b01649 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 15 Apr 2022 16:09:26 +0200 Subject: [PATCH 308/512] split out method to retrieve the download path --- .../ripme/ripper/AbstractRipper.java | 43 ++++++++++--------- .../ripper/rippers/DuckmoviesRipper.java | 2 +- .../ripme/tst/AbstractRipperTest.java | 10 ++--- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index ea713789..4321d337 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -9,6 +9,7 @@ import java.io.IOException; import java.lang.reflect.Constructor; import java.net.MalformedURLException; import java.net.URL; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -317,31 +318,18 @@ public abstract class AbstractRipper return false; } LOGGER.debug("url: " + url + ", subdirectory" + subdirectory + ", referrer: " + referrer + ", cookies: " + cookies + ", prefix: " + prefix + ", fileName: " + fileName); - String saveAs = getFileName(url, prefix, fileName, extension); - File saveFileAs; + Path saveAs; try { - if (!subdirectory.equals("")) { - subdirectory = Utils.filesystemSafe(subdirectory); - subdirectory = File.separator + subdirectory; + saveAs = getFilePath(url, subdirectory, prefix, fileName, extension); + LOGGER.debug("Downloading " + url + " to " + saveAs); + if (!Files.exists(saveAs.getParent())) { + LOGGER.info("[+] Creating directory: " + saveAs.getParent()); + Files.createDirectories(saveAs.getParent()); } - String topFolderName = workingDir.getCanonicalPath(); - if (App.stringToAppendToFoldername != null) { - topFolderName = topFolderName + App.stringToAppendToFoldername; - } - saveFileAs = new File( - topFolderName - + subdirectory - + File.separator - + saveAs); } catch (IOException e) { LOGGER.error("[!] Error creating save file path for URL '" + url + "':", e); return false; } - LOGGER.debug("Downloading " + url + " to " + saveFileAs); - if (!saveFileAs.getParentFile().exists()) { - LOGGER.info("[+] Creating directory: " + saveFileAs.getParent()); - saveFileAs.getParentFile().mkdirs(); - } if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) { LOGGER.info("Writing " + url.toExternalForm() + " to file"); try { @@ -350,7 +338,7 @@ public abstract class AbstractRipper LOGGER.debug("Unable to write URL history file"); } } - return addURLToDownload(url, saveFileAs.toPath(), referrer, cookies, getFileExtFromMIME); + return addURLToDownload(url, saveAs, referrer, cookies, getFileExtFromMIME); } protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map cookies, String fileName, String extension) { @@ -393,6 +381,21 @@ public abstract class AbstractRipper return addURLToDownload(url, prefix, ""); } + public Path getFilePath(URL url, String subdir, String prefix, String fileName, String extension) throws IOException { + // construct the path: workingdir + subdir + prefix + filename + extension + // save into working dir + Path filepath = Paths.get(workingDir.getCanonicalPath()); + + if (null != App.stringToAppendToFoldername) + filepath = filepath.resolveSibling(filepath.getFileName() + App.stringToAppendToFoldername); + + if (null != subdir && !subdir.trim().isEmpty()) + filepath = filepath.resolve(Utils.filesystemSafe(subdir)); + + filepath = filepath.resolve(getFileName(url, prefix, fileName, extension)); + return filepath; + } + public static String getFileName(URL url, String prefix, String fileName, String extension) { // retrieve filename from URL if not passed if (fileName == null || fileName.trim().isEmpty()) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DuckmoviesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DuckmoviesRipper.java index 48c1856c..696ea015 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DuckmoviesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DuckmoviesRipper.java @@ -131,7 +131,7 @@ public class DuckmoviesRipper extends AbstractSingleFileRipper { @Override public void downloadURL(URL url, int index) { - addURLToDownload(url, "", "", null, null, AbstractRipper.getFileName(url, null, null).replaceAll("%20", "_")); + addURLToDownload(url, "", "", null, null, null); } @Override diff --git a/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java b/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java index a388151c..c750b22b 100644 --- a/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java @@ -13,19 +13,19 @@ public class AbstractRipperTest { @Test public void testGetFileName() throws IOException { - String fileName = AbstractRipper.getFileName(new URL("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D"), "test", "test"); + String fileName = AbstractRipper.getFileName(new URL("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D"),null, "test", "test"); assertEquals("test.test", fileName); - fileName = AbstractRipper.getFileName(new URL("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D"), "test", null); + fileName = AbstractRipper.getFileName(new URL("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D"), null,"test", null); assertEquals("test", fileName); - fileName = AbstractRipper.getFileName(new URL("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D"), null, null); + fileName = AbstractRipper.getFileName(new URL("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D"), null,null, null); assertEquals("Object", fileName); - fileName = AbstractRipper.getFileName(new URL("http://www.test.com/file.png"), null, null); + fileName = AbstractRipper.getFileName(new URL("http://www.test.com/file.png"), null,null, null); assertEquals("file.png", fileName); - fileName = AbstractRipper.getFileName(new URL("http://www.test.com/file."), null, null); + fileName = AbstractRipper.getFileName(new URL("http://www.test.com/file."), null,null, null); assertEquals("file.", fileName); } From 9a683538f7d4571dfbeab53d9ae343ff06121c39 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 15 Apr 2022 18:11:23 +0200 Subject: [PATCH 309/512] no whitespace at the end of a filename --- src/main/java/com/rarchives/ripme/utils/Utils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index f3b20a5f..3038eacc 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -474,7 +474,7 @@ public class Utils { * @return a filesystem safe string */ public static String filesystemSafe(String text) { - text = text.replaceAll("[^a-zA-Z0-9-.,_ ]", ""); + text = text.replaceAll("[^a-zA-Z0-9-.,_ ]", "").trim(); if (text.length() > 100) { text = text.substring(0, 99); } From 4744d649ff67d45ce2d053c0c640b180af06967d Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 15 Apr 2022 19:38:28 +0200 Subject: [PATCH 310/512] latest version 2.1.1 for download --- ripme.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ripme.json b/ripme.json index 858c65cb..681097d3 100644 --- a/ripme.json +++ b/ripme.json @@ -1,8 +1,9 @@ { - "latestVersion": "2.0.4-13-03e32cb7", - "currentHash": "2d2437911a63f1bc75ed4b761a4cb464bd14f84dea5dab19b122bb35905381b2", + "latestVersion": "2.1.1-3-536339dd", + "currentHash": "34fef6b75740421912098d2e703f7bd5c7b602250270ceaacbb02ee5af8c0655", "changeList": [ - "2.0.4-13-03e32cb7.: fix vsco, add danbooru.", + "2.1.1-3-536339dd: java-11+ necessary to run, work around non existing working directory.", + "2.0.4-13-03e32cb7: fix vsco, add danbooru.", "2.0.3: Check new version against ripme2app.", "2.0.2: Add greek translation, fixed reddit, redgif.", "2.0.1: Fixed reddit, tujigu, xhamster, imagebam, erome; marked some tests as flaky.", From b6ba9786c2a4a0c04387c9fb2034df78f90f234e Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 16 Apr 2022 02:06:10 +0200 Subject: [PATCH 311/512] let gitter stick out a little more --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e4afcabe..93e2c748 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ![alt Badge Status](https://github.com/ripmeapp2/ripme/actions/workflows/gradle.yml/badge.svg) [![Coverage Status](https://coveralls.io/repos/github/RipMeApp/ripme/badge.svg?branch=master)](https://coveralls.io/github/RipMeApp/ripme?branch=master) -RipMe is maintained with ♥️ and in our limited free time by **[@MetaPrime](https://github.com/metaprime)**, **[@cyian-1756](https://github.com/cyian-1756)** and **[@kevin51jiang](https://github.com/kevin51jiang)**. If you'd like to contribute but aren't good with code, help keep us happy with a small contribution! +RipMe is maintained with ♥️ and in our limited free time by **[@MetaPrime](https://github.com/metaprime)**, **[@cyian-1756](https://github.com/cyian-1756)** and **[@kevin51jiang](https://github.com/kevin51jiang)**. If you'd like to contribute but aren't good with code, help keep us happy with a small contribution! Chat on [gitter](https://gitter.im/RipMeApp/Lobby). [![Tip with PayPal](https://img.shields.io/badge/PayPal-Buy_us...-lightgrey.svg)](https://www.paypal.me/ripmeapp) [![Tip with PayPal](https://img.shields.io/badge/coffee-%245-green.svg)](https://www.paypal.com/paypalme/ripmeapp/send?amount=5.00¤cyCode=USD&locale.x=en_US&country.x=US) From 9ef0f78617b3167dd17406063dc5675800e0cda8 Mon Sep 17 00:00:00 2001 From: Zsombor Gegesy Date: Wed, 13 Apr 2022 00:11:04 +0200 Subject: [PATCH 312/512] Automatically update the configuration when the value has been changed --- .../com/rarchives/ripme/ui/MainWindow.java | 39 +++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 96208632..1916053c 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -500,9 +500,9 @@ public final class MainWindow implements Runnable, RipStatusHandler { configThreadsLabel = new JLabel(Utils.getLocalizedString("max.download.threads") + ":", JLabel.RIGHT); configTimeoutLabel = new JLabel(Utils.getLocalizedString("timeout.mill"), JLabel.RIGHT); configRetriesLabel = new JLabel(Utils.getLocalizedString("retry.download.count"), JLabel.RIGHT); - configThreadsText = new JTextField(Integer.toString(Utils.getConfigInteger("threads.size", 3))); - configTimeoutText = new JTextField(Integer.toString(Utils.getConfigInteger("download.timeout", 60000))); - configRetriesText = new JTextField(Integer.toString(Utils.getConfigInteger("download.retries", 3))); + configThreadsText = configField("threads.size", 3); + configTimeoutText = configField("download.timeout", 60000); + configRetriesText = configField("download.retries", 3); configOverwriteCheckbox = addNewCheckbox(Utils.getLocalizedString("overwrite.existing.files"), "file.overwrite", false); configAutoupdateCheckbox = addNewCheckbox(Utils.getLocalizedString("auto.update"), "auto.update", true); @@ -587,6 +587,39 @@ public final class MainWindow implements Runnable, RipStatusHandler { gbc.fill = GridBagConstraints.HORIZONTAL; } + private JTextField configField(String key, int defaultValue) { + final var field = new JTextField(Integer.toString(Utils.getConfigInteger(key, defaultValue))); + field.getDocument().addDocumentListener(new DocumentListener() { + + @Override + public void insertUpdate(DocumentEvent e) { + checkAndUpdate(); + } + + @Override + public void removeUpdate(DocumentEvent e) { + checkAndUpdate(); + } + + @Override + public void changedUpdate(DocumentEvent e) { + checkAndUpdate(); + } + + private void checkAndUpdate() { + final var txt = field.getText(); + try { + final var newValue = Integer.parseInt(txt); + if (newValue>0) { + Utils.setConfigInteger(key, newValue); + } + } catch (final Exception e) { + } + } + }); + return field; + } + private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JLabel thing1ToAdd, JButton thing2ToAdd) { gbc.gridy = gbcYValue; From 01fc7acdf49170e7295f6a7d498e37a0ad6298fb Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 22 Apr 2022 08:07:42 +0200 Subject: [PATCH 313/512] deviantart test or ripper broken --- .../rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java index 33003e5d..73982c9f 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java @@ -37,6 +37,7 @@ public class DeviantartRipperTest extends RippersTest { } @Test + @Disabled("Broken ripper") public void testGetGalleryIDAndUsername() throws IOException { URL url = new URL("https://www.deviantart.com/airgee/gallery/"); DeviantartRipper ripper = new DeviantartRipper(url); From f728bed5d2fac7607a869dfc91bef66b80c6ba61 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 22 Apr 2022 08:41:03 +0200 Subject: [PATCH 314/512] reddit sanitize filename, fixes #82 sanitize now does not care about / in filenames any more, it is a subdirectory then. in case some pieces of code rely on replacing a /, fix it later and different. --- .../com/rarchives/ripme/ripper/rippers/RedditRipper.java | 9 ++++----- src/main/java/com/rarchives/ripme/utils/Utils.java | 6 +++++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 0a0fa306..4aabe559 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -6,7 +6,6 @@ import java.net.MalformedURLException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Date; import java.util.List; import java.util.regex.Matcher; @@ -287,9 +286,9 @@ public class RedditRipper extends AlbumRipper { ).renderFormatted(); try { - saveFileAs = Paths.get(workingDir + saveFileAs = Utils.getPath(workingDir + "/" - + id + "_" + title.replaceAll("[\\\\/:*?\"<>|]", "") + + id + "_" + title + ".html"); OutputStream out = Files.newOutputStream(saveFileAs); out.write(html.getBytes()); @@ -406,7 +405,7 @@ public class RedditRipper extends AlbumRipper { // It's from reddituploads. Assume .jpg extension. String savePath = this.workingDir + "/"; savePath += id + "-" + m.group(1) + title + ".jpg"; - addURLToDownload(urls.get(0), Paths.get(savePath)); + addURLToDownload(urls.get(0), Utils.getPath(savePath)); } if (url.contains("v.redd.it")) { String savePath = this.workingDir + "/"; @@ -414,7 +413,7 @@ public class RedditRipper extends AlbumRipper { URL urlToDownload = parseRedditVideoMPD(urls.get(0).toExternalForm()); if (urlToDownload != null) { LOGGER.info("url: " + urlToDownload + " file: " + savePath); - addURLToDownload(urlToDownload, Paths.get(savePath)); + addURLToDownload(urlToDownload, Utils.getPath(savePath)); } } else { diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 3038eacc..97b2f752 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -830,8 +830,12 @@ public class Utils { return Files.exists(folder.resolve(filename)); } + public static Path getPath(String pathToSanitize) { + return Paths.get(sanitizeSaveAs(pathToSanitize)); + } + public static String sanitizeSaveAs(String fileNameToSan) { - return fileNameToSan.replaceAll("[\\\\/:*?\"<>|]", "_"); + return fileNameToSan.replaceAll("[\\\\:*?\"<>|]", "_"); } public static Path shortenSaveAsWindows(String ripsDirPath, String fileName) throws FileNotFoundException { From bfc915e57d6401b6822f643e8d2aec6ef4c1d50f Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 22 Apr 2022 08:47:07 +0200 Subject: [PATCH 315/512] file.separator only important when printing for windows shell --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 1916053c..a17972b6 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1178,7 +1178,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { } private void loadHistory() throws IOException { - File historyFile = new File(Utils.getConfigDir() + File.separator + "history.json"); + File historyFile = new File(Utils.getConfigDir() + "/history.json"); HISTORY.clear(); if (historyFile.exists()) { try { @@ -1214,7 +1214,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { } private void saveHistory() { - Path historyFile = Paths.get(Utils.getConfigDir() + File.separator + "history.json"); + Path historyFile = Paths.get(Utils.getConfigDir() + "/history.json"); try { if (!Files.exists(historyFile)) { Files.createDirectories(historyFile.getParent()); From ebac12fb4c8024b9c9d3bd13378de20cd1ab9efd Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 22 Apr 2022 09:16:43 +0200 Subject: [PATCH 316/512] 2.1.2 release --- ripme.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ripme.json b/ripme.json index 681097d3..bbde4a02 100644 --- a/ripme.json +++ b/ripme.json @@ -1,7 +1,8 @@ { - "latestVersion": "2.1.1-3-536339dd", + "latestVersion": "2.1.1-21-bfc915e5", "currentHash": "34fef6b75740421912098d2e703f7bd5c7b602250270ceaacbb02ee5af8c0655", "changeList": [ + "2.1.1-21-bfc915e5: better sanitize filenames for windows, save config on update value.", "2.1.1-3-536339dd: java-11+ necessary to run, work around non existing working directory.", "2.0.4-13-03e32cb7: fix vsco, add danbooru.", "2.0.3: Check new version against ripme2app.", From d22c9bf5ead578939d68886cec71cdc2a912edd6 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 24 Apr 2022 19:28:46 +0200 Subject: [PATCH 317/512] reddit, report exception in loop instead of throwing it. fixes #60, #83. --- .../ripme/ripper/rippers/RedditRipper.java | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 4aabe559..f03511a1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -112,14 +112,18 @@ public class RedditRipper extends AlbumRipper { } children = data.getJSONArray("children"); for (int j = 0; j < children.length(); j++) { - parseJsonChild(children.getJSONObject(j)); + try { + parseJsonChild(children.getJSONObject(j)); - if (children.getJSONObject(j).getString("kind").equals("t3") && - children.getJSONObject(j).getJSONObject("data").getBoolean("is_self") - ) { - URL selfPostURL = new URL(children.getJSONObject(j).getJSONObject("data").getString("url")); - System.out.println(selfPostURL.toExternalForm()); - saveText(getJsonArrayFromURL(getJsonURL(selfPostURL))); + if (children.getJSONObject(j).getString("kind").equals("t3") && + children.getJSONObject(j).getJSONObject("data").getBoolean("is_self") + ) { + URL selfPostURL = new URL(children.getJSONObject(j).getJSONObject("data").getString("url")); + System.out.println(selfPostURL.toExternalForm()); + saveText(getJsonArrayFromURL(getJsonURL(selfPostURL))); + } + } catch (Exception e) { + LOGGER.debug("at index " + i + ", for this data: " + data.toString() + e); } } if (data.has("after") && !data.isNull("after")) { @@ -316,15 +320,18 @@ public class RedditRipper extends AlbumRipper { for (int i = 0; i < comments.length(); i++) { JSONObject data = comments.getJSONObject(i).getJSONObject("data"); - ContainerTag commentDiv = + try { + ContainerTag commentDiv = div( span(data.getString("author")).withClasses("author", iff(data.getString("author").equals(author), "op")), a(new Date((long) data.getInt("created") * 1000).toString()).withHref("#" + data.getString("name")) ).withClass("thing comment").withId(data.getString("name")) .with(rawHtml(Jsoup.parse(data.getString("body_html")).text())); - - getNestedComments(data, commentDiv, author); - commentsDiv.with(commentDiv); + getNestedComments(data, commentDiv, author); + commentsDiv.with(commentDiv); + } catch (Exception e) { + LOGGER.debug("at index " + i + ", for this data: " + data.toString() + e); + } } return commentsDiv; } From ea90b172d1c3ae6a511342b4b4a0ba48d8d6205a Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 24 Apr 2022 20:12:14 +0200 Subject: [PATCH 318/512] reddit, test flaky on windows --- .../com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java index 1641430d..b6eae52b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java @@ -48,6 +48,7 @@ public class RedditRipperTest extends RippersTest { } @Test + @Tag("flaky") public void testSelfPostRip() throws IOException { RedditRipper ripper = new RedditRipper( new URL("https://www.reddit.com/r/gonewildstories/comments/oz7d97/f_18_finally_having_a_normal_sex_life/") From bc59d9057901c77e839c3e423c6c8534270cba40 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 24 Apr 2022 20:41:12 +0200 Subject: [PATCH 319/512] 2.1.2 release --- ripme.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ripme.json b/ripme.json index bbde4a02..4a5147ff 100644 --- a/ripme.json +++ b/ripme.json @@ -1,8 +1,8 @@ { - "latestVersion": "2.1.1-21-bfc915e5", - "currentHash": "34fef6b75740421912098d2e703f7bd5c7b602250270ceaacbb02ee5af8c0655", + "latestVersion": "2.1.2-3-ea90b172", + "currentHash": "208e3a63d8bcf0d49679dd4e5b900a0a47eb5da5293e92065e19c2a056f564ee", "changeList": [ - "2.1.1-21-bfc915e5: better sanitize filenames for windows, save config on update value.", + "2.1.2-3-ea90b172: better sanitize filenames for windows, save config on update value. reddit, print exceptions in loops and continue.", "2.1.1-3-536339dd: java-11+ necessary to run, work around non existing working directory.", "2.0.4-13-03e32cb7: fix vsco, add danbooru.", "2.0.3: Check new version against ripme2app.", From 886782d4cd0519d25f24f88f9b9bfaedd9b2abbc Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 25 Apr 2022 07:23:12 +0200 Subject: [PATCH 320/512] print exceptions in main window --- .../com/rarchives/ripme/ui/MainWindow.java | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index a17972b6..4014e228 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -131,7 +131,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { model = queueListModel; if (model.size() > 0) { - Utils.setConfigList("queue", (Enumeration) model.elements()); + Utils.setConfigList("queue", model.elements()); Utils.saveConfig(); } @@ -258,6 +258,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { try { setupTrayIcon(); } catch (Exception e) { + LOGGER.warn(e.getMessage()); } EmptyBorder emptyBorder = new EmptyBorder(5, 5, 5, 5); @@ -347,6 +348,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { icon = ImageIO.read(getClass().getClassLoader().getResource("gear.png")); optionConfiguration.setIcon(new ImageIcon(icon)); } catch (Exception e) { + LOGGER.warn(e.getMessage()); } gbc.gridx = 0; optionsPanel.add(optionLog, gbc); @@ -539,6 +541,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { configSaveDirLabel.setForeground(Color.BLUE); configSaveDirLabel.setCursor(new Cursor(Cursor.HAND_CURSOR)); } catch (Exception e) { + LOGGER.error(e); } configSaveDirLabel.setToolTipText(configSaveDirLabel.getText()); configSaveDirLabel.setHorizontalAlignment(JLabel.RIGHT); @@ -614,6 +617,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { Utils.setConfigInteger(key, newValue); } } catch (final Exception e) { + LOGGER.warn(e.getMessage()); } } }); @@ -821,6 +825,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { try { historyTableModel.fireTableDataChanged(); } catch (Exception e) { + LOGGER.warn(e.getMessage()); } saveHistory(); }); @@ -852,6 +857,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { try { historyTableModel.fireTableDataChanged(); } catch (Exception e) { + LOGGER.warn(e.getMessage()); } saveHistory(); }); @@ -861,6 +867,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { try { historyTableModel.fireTableDataChanged(); } catch (Exception e) { + LOGGER.warn(e.getMessage()); } saveHistory(); } @@ -908,6 +915,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { Desktop desktop = Desktop.getDesktop(); desktop.open(file.toFile()); } catch (IOException ex) { + LOGGER.warn(ex.getMessage()); } } }); @@ -998,6 +1006,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { } private void setLogLevel(String level) { + // default level is error, set in case something else is given. Level newLevel = Level.ERROR; level = level.substring(level.lastIndexOf(' ') + 1); switch (level) { @@ -1009,10 +1018,6 @@ public final class MainWindow implements Runnable, RipStatusHandler { break; case "Warn": newLevel = Level.WARN; - break; - case "Error": - newLevel = Level.ERROR; - break; } LoggerContext ctx = (LoggerContext) LogManager.getContext(false); Configuration config = ctx.getConfiguration(); @@ -1066,6 +1071,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { } about.append(""); } catch (Exception e) { + LOGGER.warn(e.getMessage()); } about.append("
And download videos from video sites:"); try { @@ -1082,6 +1088,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { } about.append(""); } catch (Exception e) { + LOGGER.warn(e.getMessage()); } about.append("Do you want to visit the project homepage on Github?"); @@ -1130,7 +1137,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { } catch (IOException | AWTException e) { // TODO implement proper stack trace handling this is really just intented as a // placeholder until you implement proper error handling - e.printStackTrace(); + LOGGER.warn(e.getMessage()); } } @@ -1161,6 +1168,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { sd.insertString(sd.getLength(), text + "\n", sas); } } catch (BadLocationException e) { + LOGGER.warn(e.getMessage()); } logText.setCaretPosition(sd.getLength()); @@ -1451,6 +1459,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { try { entry.title = ripper.getAlbumTitle(ripper.getURL()); } catch (MalformedURLException e) { + LOGGER.warn(e.getMessage()); } HISTORY.add(entry); historyTableModel.fireTableDataChanged(); @@ -1471,6 +1480,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { Image folderIcon = ImageIO.read(getClass().getClassLoader().getResource("folder.png")); openButton.setIcon(new ImageIcon(folderIcon)); } catch (Exception e) { + LOGGER.warn(e.getMessage()); } /* * content key %path% the path to the album folder %url% is the album url From 78af90a25bb5728eae4d6a7fce5b4c36633edf7c Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 25 Apr 2022 07:32:21 +0200 Subject: [PATCH 321/512] main window, localize one variable, remove unused method --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 4014e228..669d3071 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -85,7 +85,6 @@ public final class MainWindow implements Runnable, RipStatusHandler { public static JButton optionQueue; private static JPanel queuePanel; private static DefaultListModel queueListModel; - private static QueueMenuMouseListener queueMenuMouseListener; // Configuration private static JButton optionConfiguration; @@ -472,6 +471,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { queueListModel = new DefaultListModel(); JList queueList = new JList(queueListModel); queueList.setSelectionMode(ListSelectionModel.MULTIPLE_INTERVAL_SELECTION); + QueueMenuMouseListener queueMenuMouseListener; queueList.addMouseListener( queueMenuMouseListener = new QueueMenuMouseListener(d -> updateQueue(queueListModel))); JScrollPane queueListScroll = new JScrollPane(queueList, JScrollPane.VERTICAL_SCROLLBAR_AS_NEEDED, @@ -902,7 +902,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { setLogLevel(level); }); configSelectLangComboBox.addActionListener(arg0 -> { - String level = ((JComboBox) arg0.getSource()).getSelectedItem().toString(); + String level = ((JComboBox) arg0.getSource()).getSelectedItem().toString(); Utils.setLanguage(level); changeLocale(); }); @@ -1563,10 +1563,6 @@ public final class MainWindow implements Runnable, RipStatusHandler { Utils.setConfigBoolean("window.position", true); } - public static void disableWindowPositioning() { - Utils.setConfigBoolean("window.position", false); - } - private static boolean hasWindowPositionBug() { String osName = System.getProperty("os.name"); // Java on Windows has a bug where if we try to manually set the position of the From ea097ad669ab5899a941ed3b5eb2cbde01fc1a17 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 25 Apr 2022 07:45:59 +0200 Subject: [PATCH 322/512] simplify main window variable assignments --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 669d3071..8ecbc280 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -847,9 +847,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { frame.setSize(405, 70); frame.setVisible(true); frame.setLocationRelativeTo(null); - noButton.addActionListener(e -> { - frame.setVisible(false); - }); + noButton.addActionListener(e -> frame.setVisible(false)); yesButton.addActionListener(ed -> { frame.setVisible(false); Utils.clearURLHistory(); @@ -898,7 +896,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { t.start(); }); configLogLevelCombobox.addActionListener(arg0 -> { - String level = ((JComboBox) arg0.getSource()).getSelectedItem().toString(); + String level = ((JComboBox) arg0.getSource()).getSelectedItem().toString(); setLogLevel(level); }); configSelectLangComboBox.addActionListener(arg0 -> { @@ -909,7 +907,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { configSaveDirLabel.addMouseListener(new MouseAdapter() { @Override public void mouseClicked(MouseEvent e) { - Path file = null; + Path file; try { file = Utils.getWorkingDirectory(); Desktop desktop = Desktop.getDesktop(); @@ -1239,7 +1237,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { private void ripNextAlbum() { isRipping = true; // Save current state of queue to configuration. - Utils.setConfigList("queue", (Enumeration) queueListModel.elements()); + Utils.setConfigList("queue", queueListModel.elements()); if (queueListModel.isEmpty()) { // End of queue From e3550d6f6830ed4bd39064daf75af3e87c971f45 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 25 Apr 2022 07:47:55 +0200 Subject: [PATCH 323/512] remove unused in main window --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 8ecbc280..dde03dc5 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -926,7 +926,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { return; } File chosenFile = jfc.getSelectedFile(); - String chosenPath = null; + String chosenPath; try { chosenPath = chosenFile.getCanonicalPath(); } catch (Exception e) { @@ -945,7 +945,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { return; } File chosenFile = jfc.getSelectedFile(); - String chosenPath = null; + String chosenPath; try { chosenPath = chosenFile.getCanonicalPath(); } catch (Exception e) { @@ -1273,7 +1273,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { if (!urlString.startsWith("http")) { urlString = "http://" + urlString; } - URL url = null; + URL url; try { url = new URL(urlString); } catch (MalformedURLException e) { @@ -1557,10 +1557,6 @@ public final class MainWindow implements Runnable, RipStatusHandler { ripButton.doClick(); } - public static void enableWindowPositioning() { - Utils.setConfigBoolean("window.position", true); - } - private static boolean hasWindowPositionBug() { String osName = System.getProperty("os.name"); // Java on Windows has a bug where if we try to manually set the position of the From e0e58c1435411634730ea048b18456edeb85883c Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 25 Apr 2022 09:01:42 +0200 Subject: [PATCH 324/512] shorten path uses nio, fixes #85. --- .../java/com/rarchives/ripme/ui/MainWindow.java | 12 ++++++------ .../java/com/rarchives/ripme/ui/UpdateUtils.java | 5 ++--- .../java/com/rarchives/ripme/utils/Utils.java | 16 +++++++++------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index dde03dc5..cda82eb0 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -536,7 +536,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { setLogLevel(configLogLevelCombobox.getSelectedItem().toString()); configSaveDirLabel = new JLabel(); try { - String workingDir = (Utils.shortenPath(Utils.getWorkingDirectory().toString())); + String workingDir = (Utils.shortenPath(Utils.getWorkingDirectory())); configSaveDirLabel.setText(workingDir); configSaveDirLabel.setForeground(Color.BLUE); configSaveDirLabel.setCursor(new Cursor(Cursor.HAND_CURSOR)); @@ -920,21 +920,21 @@ public final class MainWindow implements Runnable, RipStatusHandler { configSaveDirButton.addActionListener(arg0 -> { UIManager.put("FileChooser.useSystemExtensionHiding", false); JFileChooser jfc = new JFileChooser(Utils.getWorkingDirectory().toString()); + LOGGER.debug("select save directory, current is:" + Utils.getWorkingDirectory()); jfc.setFileSelectionMode(JFileChooser.DIRECTORIES_ONLY); int returnVal = jfc.showDialog(null, "select directory"); if (returnVal != JFileChooser.APPROVE_OPTION) { return; } - File chosenFile = jfc.getSelectedFile(); - String chosenPath; + Path chosenPath; try { - chosenPath = chosenFile.getCanonicalPath(); + chosenPath = jfc.getSelectedFile().toPath(); } catch (Exception e) { LOGGER.error("Error while getting selected path: ", e); return; } configSaveDirLabel.setText(Utils.shortenPath(chosenPath)); - Utils.setConfigString("rips.directory", chosenPath); + Utils.setConfigString("rips.directory", chosenPath.toString()); }); configUrlFileChooserButton.addActionListener(arg0 -> { UIManager.put("FileChooser.useSystemExtensionHiding", false); @@ -1471,7 +1471,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { statusProgress.setVisible(false); openButton.setVisible(true); Path f = rsc.dir; - String prettyFile = Utils.shortenPath(f.toFile()); + String prettyFile = Utils.shortenPath(f); openButton.setText(Utils.getLocalizedString("open") + prettyFile); mainFrame.setTitle("RipMe v" + UpdateUtils.getThisJarVersion()); try { diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 5ceb361b..29c86adc 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -168,11 +168,10 @@ public class UpdateUtils { logger.error("Error while updating: ", e); } } else { - logger.debug("This version (" + UpdateUtils.getThisJarVersion() - + ") is the same or newer than the website's version (" + latestVersion + ")"); + logger.info("Running version (" + UpdateUtils.getThisJarVersion() + + ") is not older than release (" + latestVersion + ")"); configUpdateLabel.setText("v" + UpdateUtils.getThisJarVersion() + " is the latest version"); - logger.debug("Running latest version: " + UpdateUtils.getThisJarVersion()); } } diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 97b2f752..e84e7973 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -438,21 +438,23 @@ public class Utils { * @return The simplified path to the file. */ public static String shortenPath(String path) { - return shortenPath(new File(path)); + return shortenPath(path); } /** * Shortens the path to a file * - * @param file File object that you want the shortened path of. + * @param path File object that you want the shortened path of. * @return The simplified path to the file. */ - public static String shortenPath(File file) { - String path = removeCWD(file.toPath()); - if (path.length() < SHORTENED_PATH_LENGTH * 2) { - return path; + public static String shortenPath(Path path) { + Path prettyPath = path.normalize(); + if (prettyPath.toString().length() < SHORTENED_PATH_LENGTH * 2) { + return prettyPath.toString(); } - return path.substring(0, SHORTENED_PATH_LENGTH) + "..." + path.substring(path.length() - SHORTENED_PATH_LENGTH); + return prettyPath.toString().substring(0, SHORTENED_PATH_LENGTH) + + "..." + + prettyPath.toString().substring(prettyPath.toString().length() - SHORTENED_PATH_LENGTH); } /** From 3a9f5b51f75d97c8ee87a41058d817430cb39e18 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 25 Apr 2022 09:14:53 +0200 Subject: [PATCH 325/512] clean up log messages --- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 6 ++---- src/main/java/com/rarchives/ripme/utils/Utils.java | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 29c86adc..7df775d9 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -112,10 +112,8 @@ public class UpdateUtils { logger.error("Error while updating: ", e); } } else { - logger.debug("This version (" + UpdateUtils.getThisJarVersion() - + ") is the same or newer than the website's version (" + latestVersion + ")"); - logger.info("v" + UpdateUtils.getThisJarVersion() + " is the latest version"); - logger.debug("Running latest version: " + UpdateUtils.getThisJarVersion()); + logger.info("Running version (" + UpdateUtils.getThisJarVersion() + + ") is not older than release (" + latestVersion + ")"); } } diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index e84e7973..783f28cc 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -790,7 +790,7 @@ public class Utils { } public static String getLocalizedString(String key) { - LOGGER.debug(String.format("Getting key %s in %s value %s", key, getSelectedLanguage(), + LOGGER.debug(String.format("Key %s in %s is: %s", key, getSelectedLanguage(), resourceBundle.getString(key))); return resourceBundle.getString(key); } From 7262a79a8df7bd069109f0e2e0c074b8dc7f489e Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 26 Apr 2022 02:28:36 +0200 Subject: [PATCH 326/512] read clipboard correctly to start auto-download fixes #86. --- .../rarchives/ripme/ui/ClipboardUtils.java | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java b/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java index c149d6fe..55b68d65 100644 --- a/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java @@ -1,8 +1,8 @@ package com.rarchives.ripme.ui; -import java.awt.HeadlessException; import java.awt.Toolkit; import java.awt.datatransfer.DataFlavor; +import java.awt.datatransfer.Transferable; import java.awt.datatransfer.UnsupportedFlavorException; import java.io.IOException; import java.util.HashSet; @@ -30,16 +30,13 @@ class ClipboardUtils { } public static String getClipboardString() { - try { - return (String) Toolkit - .getDefaultToolkit() - .getSystemClipboard() - .getData(DataFlavor.getTextPlainUnicodeFlavor()); - } catch (IllegalStateException e) { - e.printStackTrace(); - logger.error("Caught and recovered from IllegalStateException: " + e.getMessage()); - } catch (HeadlessException | IOException | UnsupportedFlavorException e) { - e.printStackTrace(); + Transferable contents = Toolkit.getDefaultToolkit().getSystemClipboard().getContents(null); + if (contents.isDataFlavorSupported(DataFlavor.stringFlavor)) { + try { + return (String) contents.getTransferData(DataFlavor.stringFlavor); + } catch (UnsupportedFlavorException | IOException e) { + logger.debug("ignore this one" + e.getMessage()); + } } return null; } @@ -47,7 +44,7 @@ class ClipboardUtils { class AutoripThread extends Thread { volatile boolean isRunning = false; - private Set rippedURLs = new HashSet<>(); + private final Set rippedURLs = new HashSet<>(); public void run() { isRunning = true; From 3bf05a79c7a8c8613407db20df33a3e9961440e5 Mon Sep 17 00:00:00 2001 From: Attacktive Date: Mon, 25 Apr 2022 12:03:41 +0900 Subject: [PATCH 327/512] Change behavior of closing dialog with hitting X https://github.com/RipMeApp/ripme/issues/1975 --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index cda82eb0..a20203e1 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -842,7 +842,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { checkChoise.add(noButton); JFrame.setDefaultLookAndFeelDecorated(true); JFrame frame = new JFrame("Are you sure?"); - frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); + frame.setDefaultCloseOperation(JFrame.DISPOSE_ON_CLOSE); frame.add(checkChoise); frame.setSize(405, 70); frame.setVisible(true); From 54d861af4c63f8ec2ab2970c94fb7381902751bb Mon Sep 17 00:00:00 2001 From: Stephan Martin Date: Sat, 12 Mar 2022 06:22:27 +0100 Subject: [PATCH 328/512] German language fixes Hello, this fixes some missing and some wrong german translations. --- src/main/resources/LabelsBundle_de_DE.properties | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/resources/LabelsBundle_de_DE.properties b/src/main/resources/LabelsBundle_de_DE.properties index 61461aba..9e767566 100644 --- a/src/main/resources/LabelsBundle_de_DE.properties +++ b/src/main/resources/LabelsBundle_de_DE.properties @@ -10,8 +10,8 @@ Configuration = Konfiguration current.version = Aktuelle Version check.for.updates = Suche nach Aktualisierungen auto.update = Automatisch Aktualisieren? -max.download.threads = Maximum download threads -timeout.mill = Timeout (in milliseconds): +max.download.threads = Maximale Download-Threads +timeout.mill = Timeout (in Milliseconds): retry.download.count = Anzahl der Downloadversuche overwrite.existing.files = Überschreibe bereits existierende Dateien? sound.when.rip.completes = Ton abspielen bei fertigem Download @@ -24,7 +24,7 @@ autorip.from.clipboard = Automatisch Downloaden von der Zwischenablage save.descriptions = Speichere Beschreibungen prefer.mp4.over.gif = Bevorzuge MP4 über GIF restore.window.position = Wieder herstellen der Fensterposition -remember.url.history = Erinnere URL Verlauf +remember.url.history = Speichere URL Verlauf loading.history.from = Lade Verlauf von # Misc UI keys @@ -32,11 +32,11 @@ loading.history.from = Lade Verlauf von loading.history.from.configuration = Lade Verlauf aus Konfiguration interrupted.while.waiting.to.rip.next.album = Unterbrochen während Download des nächsten Albums inactive = Inaktiv -re-rip.checked = Re-rip Überprüft +re-rip.checked = Re-rip Ausgewählte remove = Entfernen clear = Leeren -download.url.list = Download url list -select.save.dir = Select Save Directory +download.url.list = Download URL Liste +select.save.dir = Wähle Zielverzeichnis # Keys for the logs generated by DownloadFileThread @@ -54,4 +54,4 @@ http.status.exception = HTTP status exception exception.while.downloading.file = Exception while downloading file failed.to.download = Failed to download skipping = Skipping -file.already.exists = file already exists \ No newline at end of file +file.already.exists = file already exists From 825fd94ced64775033270b8767cc5e1a56f1b1e9 Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 27 Apr 2022 05:07:52 +0200 Subject: [PATCH 329/512] newgrounds no _full --- .../rarchives/ripme/ripper/rippers/NewgroundsRipper.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NewgroundsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NewgroundsRipper.java index b3ededc4..a7be157a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NewgroundsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NewgroundsRipper.java @@ -53,7 +53,7 @@ public class NewgroundsRipper extends AbstractHTMLRipper { @Override protected Document getFirstPage() throws IOException { - return Http.url("https://" + this.username + ".newgrounds.com/art").get(); + return Http.url("https://" + this.username + ".newgrounds.com/art").timeout(10*1000).get(); } @Override @@ -71,7 +71,7 @@ public class NewgroundsRipper extends AbstractHTMLRipper { List imageURLs = new ArrayList<>(); String documentHTMLString = page.toString().replaceAll(""", ""); - String findStr = "newgrounds.com\\/art\\/view\\/" + this.username; + String findStr = "newgrounds.com/art/view/" + this.username; int lastIndex = 0; // Index where findStr is found; each occasion contains the link to an image @@ -95,7 +95,7 @@ public class NewgroundsRipper extends AbstractHTMLRipper { if(i == indices.size() - 1){ s = documentHTMLString.substring(indices.get(i) + 2); } else{ - s = documentHTMLString.substring(indices.get(i) + 2, indices.get(i + 1)); + s = documentHTMLString.substring(indices.get(i) + 1, indices.get(i + 1)); } s = s.replaceAll("\n", "").replaceAll("\t", "") @@ -106,13 +106,14 @@ public class NewgroundsRipper extends AbstractHTMLRipper { if (m.lookingAt()) { String testURL = m.group(3) + "_" + this.username + "_" + m.group(1); + testURL = testURL.replace("_full", ""); // Open new document to get full sized image try { Document imagePage = Http.url(inLink + m.group(1)).get(); for(String extensions: this.ALLOWED_EXTENSIONS){ if(imagePage.toString().contains(testURL + "." + extensions)){ - imageUrl += m.group(2) + "/" + m.group(3) + "_" + this.username + "_" + m.group(1) + "." + extensions; + imageUrl += m.group(2) + "/" + m.group(3).replace("_full","") + "_" + this.username + "_" + m.group(1) + "." + extensions; imageURLs.add(imageUrl); break; } From 8b2304d491ff07c9d608abad0fb6b516be1ee9fe Mon Sep 17 00:00:00 2001 From: Kleptoc Date: Fri, 4 Mar 2022 11:46:20 -0700 Subject: [PATCH 330/512] Fix for nhentai changing image URLs --- .../java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java index 49fc1d8a..fe50f1f1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java @@ -126,7 +126,7 @@ public class NhentaiRipper extends AbstractHTMLRipper { List imageURLs = new ArrayList<>(); Elements thumbs = page.select("a.gallerythumb > img"); for (Element el : thumbs) { - imageURLs.add(el.attr("data-src").replaceAll("t\\.n", "i.n").replaceAll("t\\.", ".")); + imageURLs.add(el.attr("data-src").replaceAll("://t", "://i").replaceAll("t\\.", ".")); } return imageURLs; } From 827cc13398cd4d69903ca600c1ef0386de57da9d Mon Sep 17 00:00:00 2001 From: Kleptoc Date: Tue, 3 May 2022 12:01:07 -0700 Subject: [PATCH 331/512] Fix for IllegalArgumentException on pathing --- src/main/java/com/rarchives/ripme/utils/Utils.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 783f28cc..f0fdbd2c 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -322,7 +322,12 @@ public class Utils { * @return saveAs in relation to the CWD */ public static String removeCWD(Path saveAs) { - return saveAs.relativize(Paths.get(".").toAbsolutePath()).toString(); + try { + return saveAs.relativize(Paths.get(".").toAbsolutePath()).toString(); + } + catch (IllegalArgumentException e) { + return saveAs.toString(); + } } /** From a06ca354729ed2649e8b2ec883ce598f9f23f431 Mon Sep 17 00:00:00 2001 From: holysoles Date: Sat, 27 Aug 2022 20:08:48 +0000 Subject: [PATCH 332/512] update tkn URL --- .../com/rarchives/ripme/ripper/rippers/VscoRipper.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java index e324d5cf..8ff4c85f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java @@ -101,13 +101,10 @@ public class VscoRipper extends AbstractHTMLRipper { } private String getUserTkn(String username) { - String userinfoPage = "https://vsco.co/content/Static/userinfo"; - String referer = "https://vsco.co/" + username + "/gallery"; - Map cookies = new HashMap<>(); + String userTokenPage = "https://vsco.co/content/Static"; Map responseCookies = new HashMap<>(); - cookies.put("vs_anonymous_id", UUID.randomUUID().toString()); try { - Response resp = Http.url(userinfoPage).cookies(cookies).referrer(referer).ignoreContentType().response(); + Response resp = Http.url(userTokenPage).ignoreContentType().response(); responseCookies = resp.cookies(); return responseCookies.get("vs"); } catch (IOException e) { From d6c201bde5bc4f0fef526e6b890d2e531f5e9c11 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 8 Oct 2022 06:46:49 +0200 Subject: [PATCH 333/512] update gradle version to 7.5.1 --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index aa991fce..ae04661e 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 38751dbd4bae043d648ef3ec0810ae81a0eb10ef Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 8 Oct 2022 07:07:43 +0200 Subject: [PATCH 334/512] disable not working tests --- .../ripme/tst/ripper/rippers/EightmusesRipperTest.java | 2 ++ .../ripme/tst/ripper/rippers/GirlsOfDesireRipperTest.java | 2 ++ .../rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java | 3 --- .../ripme/tst/ripper/rippers/PorncomixinfoRipperTest.java | 2 ++ .../rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java | 4 +--- .../com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java | 4 ++++ .../rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java | 1 + .../ripme/tst/ripper/rippers/WebtoonsRipperTest.java | 2 ++ 8 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java index e3e9bcb7..70799d96 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java @@ -5,10 +5,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.EightmusesRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class EightmusesRipperTest extends RippersTest { @Test + @Tag("flaky") public void testEightmusesAlbum() throws IOException { // A simple image album EightmusesRipper ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore")); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GirlsOfDesireRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GirlsOfDesireRipperTest.java index 07fb8616..39e6b3c1 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GirlsOfDesireRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GirlsOfDesireRipperTest.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.GirlsOfDesireRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class GirlsOfDesireRipperTest extends RippersTest { @Test + @Tag("flaky") public void testGirlsofdesireAlbum() throws IOException { GirlsOfDesireRipper ripper = new GirlsOfDesireRipper(new URL("http://www.girlsofdesire.org/galleries/krillia/")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java index dc12bbe8..bb4897ba 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java @@ -9,7 +9,6 @@ public class ListalRipperTest extends RippersTest { /** * Test for list type url. - * @throws IOException */ @Test public void testPictures() throws IOException { @@ -20,7 +19,6 @@ public class ListalRipperTest extends RippersTest { /** * Test for list type url. - * @throws IOException */ @Test public void testRipListType() throws IOException { @@ -31,7 +29,6 @@ public class ListalRipperTest extends RippersTest { /** * Test for folder type url. - * @throws IOException */ @Test public void testRipFolderType() throws IOException { diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixinfoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixinfoRipperTest.java index 10f70ac0..76841add 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixinfoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixinfoRipperTest.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.PorncomixinfoRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class PorncomixinfoRipperTest extends RippersTest { @Test + @Tag("flaky") public void testRip() throws IOException { PorncomixinfoRipper ripper = new PorncomixinfoRipper(new URL("https://porncomixinfo.net/chapter/alx-come-to-naught-down-in-flames-up-in-smoke-tracy-scops/alx-come-to-naught-down-in-flames-up-in-smoke-tracy-scops/")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java index 01c7a622..b6cec8e3 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java @@ -12,7 +12,6 @@ public class RedgifsRipperTest extends RippersTest { /** * Rips correctly formatted URL directly from Redgifs - * @throws IOException */ @Test @Disabled("test or ripper broken") @@ -23,7 +22,6 @@ public class RedgifsRipperTest extends RippersTest { /** * Rips gifdeliverynetwork URL's by redirecting them to proper redgifs url - * @throws IOException */ @Test @Tag("flaky") @@ -34,7 +32,6 @@ public class RedgifsRipperTest extends RippersTest { /** * Rips a Redifs profile - * @throws IOException */ @Test @Tag("flaky") @@ -60,6 +57,7 @@ public class RedgifsRipperTest extends RippersTest { } @Test + @Tag("flaky") public void testRedditRedgifs() throws IOException { RedditRipper ripper = new RedditRipper(new URL("https://www.reddit.com/r/nsfwhardcore/comments/ouz5bw/me_cumming_on_his_face/")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java index b7f52e99..2580d85c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java @@ -6,6 +6,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.VkRipper; import org.json.JSONObject; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class VkRipperTest extends RippersTest { @@ -18,17 +19,20 @@ public class VkRipperTest extends RippersTest { // EXAMPLE: https://vk.com/album45506334_00?rev=1 (a single album - wall pictures) // EXAMPLE: https://vk.com/album45506334_101886701 (a single album - custom) @Test + @Tag("flaky") public void testVkAlbumHttpRip() throws IOException { VkRipper ripper = new VkRipper(new URL("https://vk.com/album45506334_0")); testRipper(ripper); } @Test + @Tag("flaky") public void testVkPhotosRip() throws IOException { VkRipper ripper = new VkRipper(new URL("https://vk.com/photos45506334")); testRipper(ripper); } @Test + @Tag("flaky") public void testFindJSONObjectContainingPhotoID() throws IOException { VkRipper ripper = new VkRipper(new URL("http://vk.com/album45506334_0")); String json = diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java index 084d3ce6..2dd58674 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java @@ -17,6 +17,7 @@ public class VscoRipperTest extends RippersTest { * @throws IOException */ @Test + @Tag("flaky") public void testSingleImageRip() throws IOException { VscoRipper ripper = new VscoRipper(new URL("https://vsco.co/jonathangodoy/media/5d1aec76bb669a128035e98a")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java index 6f4ed2eb..bb3b0413 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java @@ -6,10 +6,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.WebtoonsRipper; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class WebtoonsRipperTest extends RippersTest { @Test + @Tag("flaky") public void testWebtoonsAlbum() throws IOException { WebtoonsRipper ripper = new WebtoonsRipper(new URL("https://www.webtoons.com/en/super-hero/unordinary/episode-103/viewer?title_no=679&episode_no=109")); testRipper(ripper); From ed0e2149935833070ac4296764d9aa17f7459265 Mon Sep 17 00:00:00 2001 From: Zsombor Gegesy Date: Wed, 13 Apr 2022 09:01:04 +0200 Subject: [PATCH 335/512] Retry sleep time --- .../com/rarchives/ripme/ui/MainWindow.java | 33 +++++++++++-------- .../java/com/rarchives/ripme/utils/Http.java | 16 ++++++++- src/main/resources/LabelsBundle.properties | 1 + 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index a20203e1..26b3fcaa 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -97,6 +97,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static JLabel configSaveDirLabel; private static JButton configSaveDirButton; private static JTextField configRetriesText; + private JTextField configRetrySleepText; private static JCheckBox configAutoupdateCheckbox; private static JComboBox configLogLevelCombobox; private static JCheckBox configURLHistoryCheckbox; @@ -114,6 +115,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static JLabel configThreadsLabel; private static JLabel configTimeoutLabel; private static JLabel configRetriesLabel; + private static JLabel configRetrySleepLabel; // This doesn't really belong here but I have no idea where else to put it private static JButton configUrlFileChooserButton; @@ -502,9 +504,12 @@ public final class MainWindow implements Runnable, RipStatusHandler { configThreadsLabel = new JLabel(Utils.getLocalizedString("max.download.threads") + ":", JLabel.RIGHT); configTimeoutLabel = new JLabel(Utils.getLocalizedString("timeout.mill"), JLabel.RIGHT); configRetriesLabel = new JLabel(Utils.getLocalizedString("retry.download.count"), JLabel.RIGHT); + configRetrySleepLabel = new JLabel(Utils.getLocalizedString("retry.sleep.mill"), JLabel.RIGHT); configThreadsText = configField("threads.size", 3); configTimeoutText = configField("download.timeout", 60000); configRetriesText = configField("download.retries", 3); + configRetrySleepText = configField("download.retry.sleep", 5000); + configOverwriteCheckbox = addNewCheckbox(Utils.getLocalizedString("overwrite.existing.files"), "file.overwrite", false); configAutoupdateCheckbox = addNewCheckbox(Utils.getLocalizedString("auto.update"), "auto.update", true); @@ -547,19 +552,21 @@ public final class MainWindow implements Runnable, RipStatusHandler { configSaveDirLabel.setHorizontalAlignment(JLabel.RIGHT); configSaveDirButton = new JButton(Utils.getLocalizedString("select.save.dir") + "..."); - addItemToConfigGridBagConstraints(gbc, 0, configUpdateLabel, configUpdateButton); - addItemToConfigGridBagConstraints(gbc, 1, configAutoupdateCheckbox, configLogLevelCombobox); - addItemToConfigGridBagConstraints(gbc, 2, configThreadsLabel, configThreadsText); - addItemToConfigGridBagConstraints(gbc, 3, configTimeoutLabel, configTimeoutText); - addItemToConfigGridBagConstraints(gbc, 4, configRetriesLabel, configRetriesText); - addItemToConfigGridBagConstraints(gbc, 5, configOverwriteCheckbox, configSaveOrderCheckbox); - addItemToConfigGridBagConstraints(gbc, 6, configPlaySound, configSaveLogs); - addItemToConfigGridBagConstraints(gbc, 7, configShowPopup, configSaveURLsOnly); - addItemToConfigGridBagConstraints(gbc, 8, configClipboardAutorip, configSaveAlbumTitles); - addItemToConfigGridBagConstraints(gbc, 9, configSaveDescriptions, configPreferMp4); - addItemToConfigGridBagConstraints(gbc, 10, configWindowPosition, configURLHistoryCheckbox); - addItemToConfigGridBagConstraints(gbc, 11, configSelectLangComboBox, configUrlFileChooserButton); - addItemToConfigGridBagConstraints(gbc, 12, configSaveDirLabel, configSaveDirButton); + var idx = 0; + addItemToConfigGridBagConstraints(gbc, idx++, configUpdateLabel, configUpdateButton); + addItemToConfigGridBagConstraints(gbc, idx++, configAutoupdateCheckbox, configLogLevelCombobox); + addItemToConfigGridBagConstraints(gbc, idx++, configThreadsLabel, configThreadsText); + addItemToConfigGridBagConstraints(gbc, idx++, configTimeoutLabel, configTimeoutText); + addItemToConfigGridBagConstraints(gbc, idx++, configRetriesLabel, configRetriesText); + addItemToConfigGridBagConstraints(gbc, idx++, configRetrySleepLabel, configRetrySleepText); + addItemToConfigGridBagConstraints(gbc, idx++, configOverwriteCheckbox, configSaveOrderCheckbox); + addItemToConfigGridBagConstraints(gbc, idx++, configPlaySound, configSaveLogs); + addItemToConfigGridBagConstraints(gbc, idx++, configShowPopup, configSaveURLsOnly); + addItemToConfigGridBagConstraints(gbc, idx++, configClipboardAutorip, configSaveAlbumTitles); + addItemToConfigGridBagConstraints(gbc, idx++, configSaveDescriptions, configPreferMp4); + addItemToConfigGridBagConstraints(gbc, idx++, configWindowPosition, configURLHistoryCheckbox); + addItemToConfigGridBagConstraints(gbc, idx++, configSelectLangComboBox, configUrlFileChooserButton); + addItemToConfigGridBagConstraints(gbc, idx++, configSaveDirLabel, configSaveDirButton); emptyPanel = new JPanel(); emptyPanel.setPreferredSize(new Dimension(0, 0)); diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index 0c76a768..200d4e3d 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -30,6 +30,7 @@ public class Http { private static final Logger logger = LogManager.getLogger(Http.class); private int retries; + private int retrySleep = 0; private final String url; private Connection connection; @@ -54,6 +55,7 @@ public class Http { private void defaultSettings() { this.retries = Utils.getConfigInteger("download.retries", 1); + this.retrySleep = Utils.getConfigInteger("download.retry.sleep", 5000); connection = Jsoup.connect(this.url); connection.userAgent(AbstractRipper.USER_AGENT); connection.method(Method.GET); @@ -210,9 +212,21 @@ public class Http { if (status == 401 || status == 403) { throw new IOException("Failed to load " + url + ": Status Code " + status + ". You might be able to circumvent this error by setting cookies for this domain", e); } + if (status == 404) { + throw new IOException("File not found " + url + ": Status Code " + status + ". ", e); + } } - logger.warn("Error while loading " + url, e); + if (retrySleep > 0 && retries >= 0) { + try { + logger.warn("Error while loading " + url + " waiting "+ retrySleep + " ms before retrying.", e); + Thread.sleep(retrySleep); + } catch (final InterruptedException e1) { + e1.printStackTrace(); + } + } else { + logger.warn("Error while loading " + url, e); + } lastException = e; } } diff --git a/src/main/resources/LabelsBundle.properties b/src/main/resources/LabelsBundle.properties index 575f4f8e..63bca351 100644 --- a/src/main/resources/LabelsBundle.properties +++ b/src/main/resources/LabelsBundle.properties @@ -13,6 +13,7 @@ auto.update = Auto-update? max.download.threads = Maximum download threads: timeout.mill = Timeout (in milliseconds): retry.download.count = Retry download count +retry.sleep.mill = Wait between retries (in milliseconds): overwrite.existing.files = Overwrite existing files? sound.when.rip.completes = Sound when rip completes preserve.order = Preserve order From e6ccfc0a349616b1714bc0d145854055ad2ba62d Mon Sep 17 00:00:00 2001 From: Zsombor Gegesy Date: Wed, 13 Apr 2022 09:01:04 +0200 Subject: [PATCH 336/512] Retry sleep time in Http and in DownloadFileThread too --- .../com/rarchives/ripme/ripper/DownloadFileThread.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 562ac366..95ab777a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -299,6 +299,15 @@ class DownloadFileThread extends Thread { observer.downloadErrored(url, Utils.getLocalizedString("failed.to.download") + " " + url.toExternalForm()); return; + } else { + final var retrySleep = Utils.getConfigInteger("download.retry.sleep", 0); + if (retrySleep > 0) { + try { + sleep(retrySleep); + } catch (final InterruptedException e) { + e.printStackTrace(); + } + } } } while (true); observer.downloadCompleted(url, saveAs.toPath()); From b72a49f6f611fab71cc7423b54fe1a404f71e497 Mon Sep 17 00:00:00 2001 From: Zsombor Gegesy Date: Thu, 4 Aug 2022 08:12:19 +0200 Subject: [PATCH 337/512] Move the sleep function to Utils --- .../com/rarchives/ripme/ripper/DownloadFileThread.java | 10 ++++------ src/main/java/com/rarchives/ripme/utils/Http.java | 8 ++------ src/main/java/com/rarchives/ripme/utils/Utils.java | 7 +++++++ 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 95ab777a..498d081a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -39,6 +39,7 @@ class DownloadFileThread extends Thread { private final int TIMEOUT; + private final int retrySleep; public DownloadFileThread(URL url, File saveAs, AbstractRipper observer, Boolean getFileExtFromMIME) { super(); this.url = url; @@ -47,6 +48,7 @@ class DownloadFileThread extends Thread { this.observer = observer; this.retries = Utils.getConfigInteger("download.retries", 1); this.TIMEOUT = Utils.getConfigInteger("download.timeout", 60000); + this.retrySleep = Utils.getConfigInteger("download.retry.sleep", 0); this.getFileExtFromMIME = getFileExtFromMIME; } @@ -62,6 +64,7 @@ class DownloadFileThread extends Thread { * Attempts to download the file. Retries as needed. Notifies observers upon * completion/error/warn. */ + @Override public void run() { // First thing we make sure the file name doesn't have any illegal chars in it saveAs = new File( @@ -300,13 +303,8 @@ class DownloadFileThread extends Thread { Utils.getLocalizedString("failed.to.download") + " " + url.toExternalForm()); return; } else { - final var retrySleep = Utils.getConfigInteger("download.retry.sleep", 0); if (retrySleep > 0) { - try { - sleep(retrySleep); - } catch (final InterruptedException e) { - e.printStackTrace(); - } + Utils.sleep(retrySleep); } } } while (true); diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index 200d4e3d..35051846 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -218,12 +218,8 @@ public class Http { } if (retrySleep > 0 && retries >= 0) { - try { - logger.warn("Error while loading " + url + " waiting "+ retrySleep + " ms before retrying.", e); - Thread.sleep(retrySleep); - } catch (final InterruptedException e1) { - e1.printStackTrace(); - } + logger.warn("Error while loading " + url + " waiting "+ retrySleep + " ms before retrying.", e); + Utils.sleep(retrySleep); } else { logger.warn("Error while loading " + url, e); } diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index f0fdbd2c..e15dd9cc 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -866,4 +866,11 @@ public class Utils { return Paths.get(fullPath); } + public static void sleep(long time) { + try { + Thread.sleep(time); + } catch (final InterruptedException e1) { + e1.printStackTrace(); + } + } } From e581eeee29d93e6e50203fb6389a3b69124b03cb Mon Sep 17 00:00:00 2001 From: Zsombor Gegesy Date: Thu, 4 Aug 2022 08:04:00 +0200 Subject: [PATCH 338/512] Thread -> runnable --- .../rarchives/ripme/ripper/DownloadFileThread.java | 2 +- .../rarchives/ripme/ripper/DownloadThreadPool.java | 4 ++-- .../rarchives/ripme/ripper/DownloadVideoThread.java | 13 +++++++------ .../ripme/ripper/rippers/DeviantartRipper.java | 4 ++-- .../rarchives/ripme/ripper/rippers/E621Ripper.java | 6 +++--- .../ripme/ripper/rippers/EHentaiRipper.java | 2 +- .../ripme/ripper/rippers/HentaidudeRipper.java | 2 +- .../ripme/ripper/rippers/HqpornerRipper.java | 2 +- .../ripme/ripper/rippers/ImagebamRipper.java | 6 +++--- .../ripme/ripper/rippers/ImagevenueRipper.java | 6 +++--- .../ripme/ripper/rippers/ListalRipper.java | 6 +++--- .../ripme/ripper/rippers/LusciousRipper.java | 2 +- .../ripme/ripper/rippers/MotherlessRipper.java | 6 +++--- .../rarchives/ripme/ripper/rippers/NfsfwRipper.java | 8 ++++---- .../ripme/ripper/rippers/PornhubRipper.java | 6 +++--- 15 files changed, 38 insertions(+), 37 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 498d081a..a6722971 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -24,7 +24,7 @@ import com.rarchives.ripme.utils.Utils; * Thread for downloading files. Includes retry logic, observer notifications, * and other goodies. */ -class DownloadFileThread extends Thread { +class DownloadFileThread implements Runnable { private static final Logger logger = LogManager.getLogger(DownloadFileThread.class); private String referrer = ""; diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java index e3f9e79c..8ae43743 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java @@ -35,10 +35,10 @@ public class DownloadThreadPool { } /** * For adding threads to execution pool. - * @param t + * @param t * Thread to be added. */ - public void addThread(Thread t) { + public void addThread(Runnable t) { threadPool.execute(t); } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index 001e16ec..9430adce 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -20,15 +20,15 @@ import org.apache.logging.log4j.Logger; * Thread for downloading files. * Includes retry logic, observer notifications, and other goodies. */ -class DownloadVideoThread extends Thread { +class DownloadVideoThread implements Runnable { private static final Logger logger = LogManager.getLogger(DownloadVideoThread.class); - private URL url; - private Path saveAs; - private String prettySaveAs; - private AbstractRipper observer; - private int retries; + private final URL url; + private final Path saveAs; + private final String prettySaveAs; + private final AbstractRipper observer; + private final int retries; public DownloadVideoThread(URL url, Path saveAs, AbstractRipper observer) { super(); @@ -43,6 +43,7 @@ class DownloadVideoThread extends Thread { * Attempts to download the file. Retries as needed. * Notifies observers upon completion/error/warn. */ + @Override public void run() { try { observer.stopCheck(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 99374ad1..d701f1f0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -518,8 +518,8 @@ public class DeviantartRipper extends AbstractHTMLRipper { * @author MrPlaygon * */ - private class DeviantartImageThread extends Thread { - private URL url; + private class DeviantartImageThread implements Runnable { + private final URL url; public DeviantartImageThread(URL url) { this.url = url; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java index 21e0f866..13f75f22 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java @@ -193,10 +193,10 @@ public class E621Ripper extends AbstractHTMLRipper { return url; } - public class E621FileThread extends Thread { + public class E621FileThread implements Runnable { - private URL url; - private String index; + private final URL url; + private final String index; public E621FileThread(URL url, String index) { this.url = url; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java index ba7e446d..97373e5e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java @@ -206,7 +206,7 @@ public class EHentaiRipper extends AbstractHTMLRipper { *

* Handles case when site has IP-banned the user. */ - private class EHentaiImageThread extends Thread { + private class EHentaiImageThread implements Runnable { private final URL url; private final int index; private final Path workingDir; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java index 7950f0cf..2c7ae568 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java @@ -84,7 +84,7 @@ public class HentaidudeRipper extends AbstractSingleFileRipper { return hentaidudeThreadPool; } - private class HentaidudeDownloadThread extends Thread { + private class HentaidudeDownloadThread implements Runnable { private URL url; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java index 8d13f113..17c379e3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java @@ -130,7 +130,7 @@ public class HqpornerRipper extends AbstractHTMLRipper { return true; } - private class HqpornerDownloadThread extends Thread { + private class HqpornerDownloadThread implements Runnable { private URL hqpornerVideoPageUrl; //private int index; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index e233c03d..980f62b9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -117,9 +117,9 @@ public class ImagebamRipper extends AbstractHTMLRipper { * * Handles case when site has IP-banned the user. */ - private class ImagebamImageThread extends Thread { - private URL url; //link to "image page" - private int index; //index in album + private class ImagebamImageThread implements Runnable { + private final URL url; //link to "image page" + private final int index; //index in album ImagebamImageThread(URL url, int index) { super(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java index f50a84a0..8e250ae9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java @@ -79,9 +79,9 @@ public class ImagevenueRipper extends AbstractHTMLRipper { * * Handles case when site has IP-banned the user. */ - private class ImagevenueImageThread extends Thread { - private URL url; - private int index; + private class ImagevenueImageThread implements Runnable { + private final URL url; + private final int index; ImagevenueImageThread(URL url, int index) { super(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java index fed85531..e9f6deef 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java @@ -182,10 +182,10 @@ public class ListalRipper extends AbstractHTMLRipper { throw new MalformedURLException("Unable to fetch the gid for given url."); } - private class ListalImageDownloadThread extends Thread { + private class ListalImageDownloadThread implements Runnable { - private URL url; - private int index; + private final URL url; + private final int index; public ListalImageDownloadThread(URL url, int index) { super(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java index 441fc10f..09fd8247 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java @@ -120,7 +120,7 @@ public class LusciousRipper extends AbstractHTMLRipper { } } - public class LusciousDownloadThread extends Thread { + public class LusciousDownloadThread implements Runnable { private final URL url; private final int index; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java index 598cf5d4..46331c7b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java @@ -152,9 +152,9 @@ public class MotherlessRipper extends AbstractHTMLRipper { /** * Helper class to find and download images found on "image" pages */ - private class MotherlessImageThread extends Thread { - private URL url; - private int index; + private class MotherlessImageThread implements Runnable { + private final URL url; + private final int index; MotherlessImageThread(URL url, int index) { super(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java index 86079edc..d0769138 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java @@ -196,10 +196,10 @@ public class NfsfwRipper extends AbstractHTMLRipper { /** * Helper class to find and download images found on "image" pages */ - private class NfsfwImageThread extends Thread { - private URL url; - private String subdir; - private int index; + private class NfsfwImageThread implements Runnable { + private final URL url; + private final String subdir; + private final int index; NfsfwImageThread(URL url, String subdir, int index) { super(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java index 0c0efd14..a215102e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java @@ -126,9 +126,9 @@ public class PornhubRipper extends AbstractHTMLRipper { * * Handles case when site has IP-banned the user. */ - private class PornhubImageThread extends Thread { - private URL url; - private int index; + private class PornhubImageThread implements Runnable { + private final URL url; + private final int index; PornhubImageThread(URL url, int index, Path workingDir) { super(); From 98f37208b16147a806dfc53f5c33901a57d12c10 Mon Sep 17 00:00:00 2001 From: Zsombor Gegesy Date: Wed, 3 Aug 2022 01:49:03 +0200 Subject: [PATCH 339/512] Implement caching of the first page for every ripper --- .../rarchives/ripme/ripper/AbstractHTMLRipper.java | 11 ++++++++++- .../ripme/ripper/rippers/AerisdiesRipper.java | 2 +- .../rarchives/ripme/ripper/rippers/BatoRipper.java | 2 +- .../rarchives/ripme/ripper/rippers/ChanRipper.java | 2 +- .../ripme/ripper/rippers/CheveretoRipper.java | 2 +- .../ripme/ripper/rippers/EightmusesRipper.java | 12 ++++-------- .../ripme/ripper/rippers/EroShareRipper.java | 2 +- .../ripme/ripper/rippers/EromeRipper.java | 2 +- .../ripme/ripper/rippers/FlickrRipper.java | 8 ++------ .../ripme/ripper/rippers/GirlsOfDesireRipper.java | 9 ++------- .../ripme/ripper/rippers/HbrowseRipper.java | 2 +- .../ripme/ripper/rippers/HentaifoxRipper.java | 2 +- .../ripme/ripper/rippers/ImagebamRipper.java | 10 ++-------- .../ripme/ripper/rippers/ImagefapRipper.java | 8 ++------ .../ripme/ripper/rippers/ViewcomicRipper.java | 2 +- .../ripme/ripper/rippers/XhamsterRipper.java | 2 +- .../ripme/ripper/rippers/ZizkiRipper.java | 14 +++++--------- 17 files changed, 37 insertions(+), 55 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 45378bfa..638af188 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -32,6 +32,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { private final Map itemsPending = Collections.synchronizedMap(new HashMap<>()); private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); + Document cachedFirstPage; protected AbstractHTMLRipper(URL url) throws IOException { super(url); @@ -41,6 +42,14 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { public abstract String getHost(); protected abstract Document getFirstPage() throws IOException; + + protected Document getCachedFirstPage() throws IOException { + if (cachedFirstPage == null) { + cachedFirstPage = getFirstPage(); + } + return cachedFirstPage; + } + public Document getNextPage(Document doc) throws IOException { return null; } @@ -98,7 +107,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { int textindex = 0; LOGGER.info("Retrieving " + this.url); sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm()); - Document doc = getFirstPage(); + var doc = getCachedFirstPage(); if (hasQueueSupport() && pageContainsAlbums(this.url)) { List urls = getAlbumsToQueue(doc); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/AerisdiesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/AerisdiesRipper.java index a11b08a4..f14543e9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/AerisdiesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/AerisdiesRipper.java @@ -49,7 +49,7 @@ public class AerisdiesRipper extends AbstractHTMLRipper { @Override public String getAlbumTitle(URL url) throws MalformedURLException { try { - Element el = getFirstPage().select(".headtext").first(); + Element el = getCachedFirstPage().select(".headtext").first(); if (el == null) { throw new IOException("Unable to get album title"); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java index 25491dfe..086e2be7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java @@ -73,7 +73,7 @@ public class BatoRipper extends AbstractHTMLRipper { public String getAlbumTitle(URL url) throws MalformedURLException { try { // Attempt to use album title as GID - return getHost() + "_" + getGID(url) + "_" + getFirstPage().select("title").first().text().replaceAll(" ", "_"); + return getHost() + "_" + getGID(url) + "_" + getCachedFirstPage().select("title").first().text().replaceAll(" ", "_"); } catch (IOException e) { // Fall back to default album naming convention LOGGER.info("Unable to find title at " + url); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java index 8c7aea6b..601bacae 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java @@ -104,7 +104,7 @@ public class ChanRipper extends AbstractHTMLRipper { public String getAlbumTitle(URL url) throws MalformedURLException { try { // Attempt to use album title as GID - Document doc = getFirstPage(); + Document doc = getCachedFirstPage(); try { String subject = doc.select(".post.op > .postinfo > .subject").first().text(); return getHost() + "_" + getGID(url) + "_" + subject; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java index 005ba5c7..4cba64f7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java @@ -53,7 +53,7 @@ public class CheveretoRipper extends AbstractHTMLRipper { public String getAlbumTitle(URL url) throws MalformedURLException { try { // Attempt to use album title as GID - Element titleElement = getFirstPage().select("meta[property=og:title]").first(); + Element titleElement = getCachedFirstPage().select("meta[property=og:title]").first(); String title = titleElement.attr("content"); title = title.substring(title.lastIndexOf('/') + 1); return getHost() + "_" + title.trim(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java index 26808aa0..f34fb504 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java @@ -23,7 +23,6 @@ import com.rarchives.ripme.utils.Http; public class EightmusesRipper extends AbstractHTMLRipper { - private Document albumDoc = null; private Map cookies = new HashMap<>(); // TODO put up a wiki page on using maps to store titles // the map for storing the title of each album when downloading sub albums @@ -64,7 +63,7 @@ public class EightmusesRipper extends AbstractHTMLRipper { public String getAlbumTitle(URL url) throws MalformedURLException { try { // Attempt to use album title as GID - Element titleElement = getFirstPage().select("meta[name=description]").first(); + Element titleElement = getCachedFirstPage().select("meta[name=description]").first(); String title = titleElement.attr("content"); title = title.replace("A huge collection of free porn comics for adults. Read", ""); title = title.replace("online for free at 8muses.com", ""); @@ -78,12 +77,9 @@ public class EightmusesRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - if (albumDoc == null) { - Response resp = Http.url(url).response(); - cookies.putAll(resp.cookies()); - albumDoc = resp.parse(); - } - return albumDoc; + Response resp = Http.url(url).response(); + cookies.putAll(resp.cookies()); + return resp.parse(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java index d64e9600..d4386f3c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java @@ -97,7 +97,7 @@ public class EroShareRipper extends AbstractHTMLRipper { if (!is_profile(url)) { try { // Attempt to use album title as GID - Element titleElement = getFirstPage().select("meta[property=og:title]").first(); + Element titleElement = getCachedFirstPage().select("meta[property=og:title]").first(); String title = titleElement.attr("content"); title = title.substring(title.lastIndexOf('/') + 1); return getHost() + "_" + getGID(url) + "_" + title.trim(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java index 932788f2..8e5e57d5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java @@ -71,7 +71,7 @@ public class EromeRipper extends AbstractHTMLRipper { public String getAlbumTitle(URL url) throws MalformedURLException { try { // Attempt to use album title as GID - Element titleElement = getFirstPage().select("meta[property=og:title]").first(); + Element titleElement = getCachedFirstPage().select("meta[property=og:title]").first(); String title = titleElement.attr("content"); title = title.substring(title.lastIndexOf('/') + 1); return getHost() + "_" + getGID(url) + "_" + title.trim(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java index 32088424..c7d91160 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java @@ -20,7 +20,6 @@ import org.jsoup.nodes.Element; public class FlickrRipper extends AbstractHTMLRipper { - private Document albumDoc = null; private final DownloadThreadPool flickrThreadPool; private enum UrlType { @@ -178,7 +177,7 @@ public class FlickrRipper extends AbstractHTMLRipper { } try { // Attempt to use album title as GID - Document doc = getFirstPage(); + Document doc = getCachedFirstPage(); String user = url.toExternalForm(); user = user.substring(user.indexOf("/photos/") + "/photos/".length()); user = user.substring(0, user.indexOf("/")); @@ -230,10 +229,7 @@ public class FlickrRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - if (albumDoc == null) { - albumDoc = Http.url(url).get(); - } - return albumDoc; + return Http.url(url).get(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java index 2afc79d1..6a269dca 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java @@ -16,8 +16,6 @@ import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; public class GirlsOfDesireRipper extends AbstractHTMLRipper { - // Current HTML document - private Document albumDoc = null; public GirlsOfDesireRipper(URL url) throws IOException { super(url); @@ -35,7 +33,7 @@ public class GirlsOfDesireRipper extends AbstractHTMLRipper { public String getAlbumTitle(URL url) throws MalformedURLException { try { // Attempt to use album title as GID - Document doc = getFirstPage(); + Document doc = getCachedFirstPage(); Elements elems = doc.select(".albumName"); return getHost() + "_" + elems.first().text(); } catch (Exception e) { @@ -64,10 +62,7 @@ public class GirlsOfDesireRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - if (albumDoc == null) { - albumDoc = Http.url(url).get(); - } - return albumDoc; + return Http.url(url).get(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HbrowseRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HbrowseRipper.java index fd3b23c2..34072f22 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HbrowseRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HbrowseRipper.java @@ -51,7 +51,7 @@ public class HbrowseRipper extends AbstractHTMLRipper { @Override public String getAlbumTitle(URL url) throws MalformedURLException { try { - Document doc = getFirstPage(); + Document doc = getCachedFirstPage(); String title = doc.select("div[id=main] > table.listTable > tbody > tr > td.listLong").first().text(); return getHost() + "_" + title + "_" + getGID(url); } catch (Exception e) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java index a4e5895d..a811cf9d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java @@ -61,7 +61,7 @@ public class HentaifoxRipper extends AbstractHTMLRipper { @Override public String getAlbumTitle(URL url) throws MalformedURLException { try { - Document doc = getFirstPage(); + Document doc = getCachedFirstPage(); String title = doc.select("div.info > h1").first().text(); return getHost() + "_" + title + "_" + getGID(url); } catch (Exception e) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index 980f62b9..6aeb564f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -19,9 +19,6 @@ import org.jsoup.select.Elements; public class ImagebamRipper extends AbstractHTMLRipper { - // Current HTML document - private Document albumDoc = null; - // Thread pool for finding direct image links from "image" pages (html) private DownloadThreadPool imagebamThreadPool = new DownloadThreadPool("imagebam"); @Override @@ -61,10 +58,7 @@ public class ImagebamRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - if (albumDoc == null) { - albumDoc = Http.url(url).get(); - } - return albumDoc; + return Http.url(url).get(); } @Override @@ -99,7 +93,7 @@ public class ImagebamRipper extends AbstractHTMLRipper { public String getAlbumTitle(URL url) throws MalformedURLException { try { // Attempt to use album title as GID - Elements elems = getFirstPage().select("[id=gallery-name]"); + Elements elems = getCachedFirstPage().select("[id=gallery-name]"); String title = elems.first().text(); LOGGER.info("Title text: '" + title + "'"); if (StringUtils.isNotBlank(title)) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java index 14d21aa9..87dc18a5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java @@ -17,7 +17,6 @@ import com.rarchives.ripme.utils.Http; public class ImagefapRipper extends AbstractHTMLRipper { - private Document albumDoc = null; private boolean isNewAlbumType = false; private int callsMade = 0; @@ -109,10 +108,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - if (albumDoc == null) { - albumDoc = getPageWithRetries(url); - } - return albumDoc; + return getPageWithRetries(url); } @Override @@ -162,7 +158,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { public String getAlbumTitle(URL url) throws MalformedURLException { try { // Attempt to use album title as GID - String title = getFirstPage().title(); + String title = getCachedFirstPage().title(); title = title.replace("Porn Pics & Porn GIFs", ""); title = title.replace(" ", "_"); String toReturn = getHost() + "_" + title + "_" + getGID(url); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java index 27015a06..1cdd6b7d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java @@ -34,7 +34,7 @@ public class ViewcomicRipper extends AbstractHTMLRipper { public String getAlbumTitle(URL url) throws MalformedURLException { try { // Attempt to use album title as GID - String titleText = getFirstPage().select("title").first().text(); + String titleText = getCachedFirstPage().select("title").first().text(); String title = titleText.replace("Viewcomic reading comics online for free", ""); title = title.replace("_", ""); title = title.replace("|", ""); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index cf632dcb..d9707646 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -220,7 +220,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { public String getAlbumTitle(URL url) throws MalformedURLException { try { // Attempt to use album title and username as GID - Document doc = getFirstPage(); + Document doc = getCachedFirstPage(); Element user = doc.select("a.author").first(); String username = user.text(); String path = url.getPath(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java index fb048a6d..5d95580b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java @@ -19,7 +19,6 @@ import com.rarchives.ripme.utils.Http; public class ZizkiRipper extends AbstractHTMLRipper { - private Document albumDoc = null; private Map cookies = new HashMap<>(); public ZizkiRipper(URL url) throws IOException { @@ -49,10 +48,10 @@ public class ZizkiRipper extends AbstractHTMLRipper { public String getAlbumTitle(URL url) throws MalformedURLException { try { // Attempt to use album title as GID - Element titleElement = getFirstPage().select("h1.title").first(); + Element titleElement = getCachedFirstPage().select("h1.title").first(); String title = titleElement.text(); - Element authorSpan = getFirstPage().select("span[class=creator]").first(); + Element authorSpan = getCachedFirstPage().select("span[class=creator]").first(); String author = authorSpan.select("a").first().text(); LOGGER.debug("Author: " + author); return getHost() + "_" + author + "_" + title.trim(); @@ -65,12 +64,9 @@ public class ZizkiRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - if (albumDoc == null) { - Response resp = Http.url(url).response(); - cookies.putAll(resp.cookies()); - albumDoc = resp.parse(); - } - return albumDoc; + Response resp = Http.url(url).response(); + cookies.putAll(resp.cookies()); + return resp.parse(); } @Override From e5438e85b90d8df6edb99be37a528b2b0b118030 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 8 Oct 2022 08:08:41 +0200 Subject: [PATCH 340/512] webtoonstest flaky --- .../rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java index bb3b0413..500f507e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java @@ -17,6 +17,7 @@ public class WebtoonsRipperTest extends RippersTest { testRipper(ripper); } @Test + @Tag("flaky") public void testWedramabtoonsType() throws IOException { WebtoonsRipper ripper = new WebtoonsRipper(new URL("http://www.webtoons.com/en/drama/lookism/ep-145/viewer?title_no=1049&episode_no=145")); testRipper(ripper); From 3da6515b8661cf130f94ee67c84779a71ba88632 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 8 Oct 2022 09:07:25 +0200 Subject: [PATCH 341/512] relelase 2.1.2-23-e5438e85 --- ripme.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ripme.json b/ripme.json index 4a5147ff..0117f495 100644 --- a/ripme.json +++ b/ripme.json @@ -1,7 +1,8 @@ { - "latestVersion": "2.1.2-3-ea90b172", - "currentHash": "208e3a63d8bcf0d49679dd4e5b900a0a47eb5da5293e92065e19c2a056f564ee", + "latestVersion": "2.1.2-23-e5438e85", + "currentHash": "a2309ba7ec0ffe2abca4932bd1b291cf88d0e495c233b888aa8f42c459da2ee0", "changeList": [ + "2.1.2-23-e5438e85: caching of first page, retry sleep time, nhentai fixed", "2.1.2-3-ea90b172: better sanitize filenames for windows, save config on update value. reddit, print exceptions in loops and continue.", "2.1.1-3-536339dd: java-11+ necessary to run, work around non existing working directory.", "2.0.4-13-03e32cb7: fix vsco, add danbooru.", From dffcb4f79c778cfa1501b9f58407759b198a73dc Mon Sep 17 00:00:00 2001 From: Luciano Balmaceda Date: Sun, 9 Oct 2022 12:48:51 +0200 Subject: [PATCH 342/512] Update LabelsBundle_es_ES.properties --- .../resources/LabelsBundle_es_ES.properties | 86 +++++++++++-------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/src/main/resources/LabelsBundle_es_ES.properties b/src/main/resources/LabelsBundle_es_ES.properties index c178ec79..fea84e5d 100644 --- a/src/main/resources/LabelsBundle_es_ES.properties +++ b/src/main/resources/LabelsBundle_es_ES.properties @@ -4,59 +4,73 @@ created = creado modified = modificado queue = Cola Configuration = Configuracion +open = Abrir # Keys for the Configuration menu - current.version = Version Actual check.for.updates = Buscar actualizaciones -auto.update = Auto-actualizar? -max.download.threads = Maximos procesos de descarga -timeout.mill = Timeout (in milliseconds): -retry.download.count = Numero de reintentos de descarga +auto.update = Actualizar automáticamente? +max.download.threads = Número de descargas simultáneas: +timeout.mill = Tiempo máximo de espera (milisegundos): +retry.download.count = Número de reintentos de descarga: +retry.sleep.mill = Espera entre reintentos (milisegundos): overwrite.existing.files = Sobreescribir archivos existentes? -sound.when.rip.completes = Sonar cuando el Rip termina -preserve.order = Mantener orden +sound.when.rip.completes = Notificar cuando el rip termina +preserve.order = Mantener el orden save.logs = Guardar logs -notification.when.rip.starts = Notificar cuando el Rip comienza +notification.when.rip.starts = Notificar cuando el rip comienza save.urls.only = Guardar solamente URLs -save.album.titles = Guardar titulos de albunes -autorip.from.clipboard = Autorip desde Portapapeles +save.album.titles = Guardar títulos de álbumes +autorip.from.clipboard = Autorip desde el portapapeles save.descriptions = Guardar descripciones prefer.mp4.over.gif = Preferir MP4 sobre GIF -restore.window.position = Restaurar posicion de ventana +restore.window.position = Restaurar posicion de la ventana remember.url.history = Recordar historia URL loading.history.from = Cargando historia desde # Queue keys queue.remove.all = Eliminar todos los elementos -queue.validation = ¿Esta seguro que desea eliminar todos los elementos de la lista? +queue.validation = ¿Está seguro que desea eliminar todos los elementos de la lista? queue.remove.selected = Eliminar elementos seleccionados -# Misc UI keys - -loading.history.from.configuration = Cargando historia desde la configuracion -interrupted.while.waiting.to.rip.next.album = Interrumpido esperando el Rip del proximo album -inactive = Inactivo -re-rip.checked = Re-rip marcado -remove = Quitar +# History +re-rip.checked = Re-rip Marcados +remove = Remover clear = Limpiar -download.url.list = Download url list -select.save.dir = Select Save Directory +history.check.all = Marcar Todos +history.check.none = Desmarcar Todos +history.check.selected = Marcar Seleccionados +history.uncheck.selected = Desmarcar Seleccionados +history.load.failed.warning = RipMe falló al cargar la historia de historyFile.getAbsolutePath() \n\nError: %s\n\nSi cierras RipMe los contenidos de este archivo se sobreescribirán,\nhaz un backup antes de cerrar RipMe! +history.load.none = El historial está vacío. Ripea algunos álbumes primero +history.load.none.checked = Ninguna entrada del historial fue 'marcada'. Selecciona una entrada clickeando la casilla a la derecha de la URL o haz click derecho sobre una URL para marcar/desmarcar todas las entradas. + +# TrayIcon +tray.show = Mostrar +tray.hide = Esconder +tray.autorip = Autorip desde el portapapeles +tray.exit = Salida + +# Misc UI keys +loading.history.from.configuration = Cargando historia desde la configuración +interrupted.while.waiting.to.rip.next.album = Interrumpido esperando el rip del próximo álbum +inactive = Inactivo +download.url.list = Lista de URLs a descargar +select.save.dir = Seleccione el directorio de guardado # Keys for the logs generated by DownloadFileThread - -nonretriable.status.code = Non-retriable status code -retriable.status.code = Retriable status code -server.doesnt.support.resuming.downloads = Server doesn't support resuming downloads +nonretriable.status.code = Código de estado no recuperable +retriable.status.code = Código de estado recuperable +server.doesnt.support.resuming.downloads = El servidor no soporta resumir las descargas # A "magic number" can also be called a file signature -was.unable.to.get.content.type.using.magic.number = Was unable to get content type using magic number -magic.number.was = Magic number was -deleting.existing.file = Deleting existing file -request.properties = Request properties -download.interrupted = Download interrupted -exceeded.maximum.retries = Exceeded maximum retries -http.status.exception = HTTP status exception -exception.while.downloading.file = Exception while downloading file -failed.to.download = Failed to download -skipping = Skipping -file.already.exists = file already exists \ No newline at end of file +was.unable.to.get.content.type.using.magic.number = Imposible obtener el tipo de contenido utilizando el número mágico +magic.number.was = El número mágico era +deleting.existing.file = Eliminando el archivo existente +request.properties = Propiedades del pedido +download.interrupted = Descarga interrumpida +exceeded.maximum.retries = Máximo número de reintentos excedido +http.status.exception = Error de estado HTTP +exception.while.downloading.file = Error al descargar archivo +failed.to.download = Descarga fallida +skipping = Saltando +file.already.exists = el fichero ya existe \ No newline at end of file From 568587e4326740a474eab0ede8675da162f62e9f Mon Sep 17 00:00:00 2001 From: 2omsnF <58115482+2omsnF@users.noreply.github.com> Date: Sat, 24 Sep 2022 19:35:06 -0400 Subject: [PATCH 343/512] Create NsfwXxxRipper.java --- .../ripme/ripper/rippers/NsfwXxxRipper.java | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java new file mode 100644 index 00000000..d25f453b --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java @@ -0,0 +1,124 @@ +package com.rarchives.ripme.ripper.rippers; + +import com.rarchives.ripme.ripper.AbstractJSONRipper; +import com.rarchives.ripme.utils.Http; +import org.json.JSONArray; +import org.json.JSONObject; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class NsfwXxxRipper extends AbstractJSONRipper { + + public NsfwXxxRipper(URL url) throws IOException { + super(url); + } + + @Override + protected String getDomain() { + return "nsfw.xxx"; + } + + @Override + public String getHost() { + return "nsfw_xxx"; + } + + + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + String u = url.toExternalForm(); + // https://nsfw.xxx/user/kelly-kat/foo -> https://nsfw.xxx/user/kelly-kat + // https://nsfw.xxx/user/kelly-kat -> https://nsfw.xxx/user/kelly-kat + // keep up to and including the username + u = u.replaceAll("https?://nsfw.xxx/user/([^/]+)/?.*", "https://nsfw.xxx/user/$1"); + if (!u.contains("nsfw.xxx/user")) { + throw new MalformedURLException("Invalid URL: " + url); + } + + return new URL(u); + } + + String getUser() throws MalformedURLException { + return getGID(url); + } + + URL getPage(int page) throws MalformedURLException { + return new URL("https://nsfw.xxx/slide-page/" + page + "?nsfw%5B%5D=0&types%5B%5D=image&types%5B%5D=video&types%5B%5D=gallery&slider=1&jsload=1&user=" + getUser()); + } + + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https://nsfw.xxx/user/([^/]+)/?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected URL format: " + + "nsfw.xxx/user/USER - got " + url + " instead"); + } + + + int currentPage = 1; + + @Override + protected JSONObject getFirstPage() throws IOException { + return Http.url(getPage(1)).getJSON(); + } + + List descriptions = new ArrayList<>(); + + @Override + protected JSONObject getNextPage(JSONObject doc) throws IOException { + currentPage++; + JSONObject nextPage = Http.url(getPage(doc.getInt("page") + 1)).getJSON(); + JSONArray items = nextPage.getJSONArray("items"); + if (items.length() == 0) { + throw new IOException("No more pages"); + } + return nextPage; + } + + class ApiEntry { + String srcUrl; + String author; + String title; + + public ApiEntry(String srcUrl, String author, String title) { + this.srcUrl = srcUrl; + this.author = author; + this.title = title; + } + } + + @Override + protected List getURLsFromJSON(JSONObject json) { + JSONArray items = json.getJSONArray("items"); + List data = IntStream + .range(0, items.length()) + .mapToObj(items::getJSONObject) + .map(o -> new ApiEntry(o.getString("src"), o.getString("author"), o.getString("title"))) + .collect(Collectors.toList()); + + data.forEach(e -> descriptions.add(e.title)); + return data.stream().map(e -> e.srcUrl).collect(Collectors.toList()); + } + + @Override + protected void downloadURL(URL url, int index) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + addURLToDownload(url, getPrefix(index) + descriptions.get(index - 1) + "_" , "", "", null); + } +} From aa5bd00d3fb55ecee9d30083b72043ca6efc35ef Mon Sep 17 00:00:00 2001 From: 2omsnF <58115482+2omsnF@users.noreply.github.com> Date: Sat, 24 Sep 2022 19:55:57 -0400 Subject: [PATCH 344/512] Create NsfwXxxRipperTest.java --- .../tst/ripper/rippers/NsfwXxxRipperTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/NsfwXxxRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NsfwXxxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NsfwXxxRipperTest.java new file mode 100644 index 00000000..029a8541 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NsfwXxxRipperTest.java @@ -0,0 +1,15 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.NsfwXxxRipper; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URL; + +public class NsfwXxxRipperTest extends RippersTest { + @Test + public void testNsfwXxxUser() throws IOException { + NsfwXxxRipper ripper = new NsfwXxxRipper(new URL("https://nsfw.xxx/user/smay3991")); + testRipper(ripper); + } +} From d1a5057dcc6ccbe5ae94ea6b880defe0ee91c0e7 Mon Sep 17 00:00:00 2001 From: 2omsnF <58115482+2omsnF@users.noreply.github.com> Date: Sat, 24 Sep 2022 19:58:53 -0400 Subject: [PATCH 345/512] Fix video download bugs --- .../ripme/ripper/rippers/NsfwXxxRipper.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java index d25f453b..30da2344 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractJSONRipper; import com.rarchives.ripme.utils.Http; +import org.apache.commons.lang.StringEscapeUtils; import org.json.JSONArray; import org.json.JSONObject; @@ -105,7 +106,20 @@ public class NsfwXxxRipper extends AbstractJSONRipper { List data = IntStream .range(0, items.length()) .mapToObj(items::getJSONObject) - .map(o -> new ApiEntry(o.getString("src"), o.getString("author"), o.getString("title"))) + .map(o -> { + String srcUrl; + if(o.has("src")) { + srcUrl = o.getString("src"); + } else { + // video source + Pattern videoHtmlSrcPattern = Pattern.compile("src=\"([^\"]+)\""); + Matcher matches = videoHtmlSrcPattern.matcher(o.getString("html")); + matches.find(); + srcUrl = StringEscapeUtils.unescapeHtml(matches.group(1)); + } + + return new ApiEntry(srcUrl, o.getString("author"), o.getString("title")); + }) .collect(Collectors.toList()); data.forEach(e -> descriptions.add(e.title)); @@ -114,11 +128,6 @@ public class NsfwXxxRipper extends AbstractJSONRipper { @Override protected void downloadURL(URL url, int index) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } addURLToDownload(url, getPrefix(index) + descriptions.get(index - 1) + "_" , "", "", null); } } From 07b90ddb780a5608c09b39a390fecdfeb26317ef Mon Sep 17 00:00:00 2001 From: noesterle Date: Wed, 19 Oct 2022 21:50:06 -0400 Subject: [PATCH 346/512] Added ability to create urls.txt before appending to it. --- .../java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 2 +- .../java/com/rarchives/ripme/ripper/AbstractJSONRipper.java | 2 +- src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 638af188..856d32f5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -336,7 +336,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { Path urlFile = Paths.get(this.workingDir + "/urls.txt"); String text = url.toExternalForm() + System.lineSeparator(); try { - Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); itemsCompleted.put(url, urlFile); } catch (IOException e) { LOGGER.error("Error while writing to " + urlFile, e); diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 47a13717..523a8ab8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -165,7 +165,7 @@ public abstract class AbstractJSONRipper extends AbstractRipper { Path urlFile = Paths.get(this.workingDir + "/urls.txt"); String text = url.toExternalForm() + System.lineSeparator(); try { - Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); itemsCompleted.put(url, urlFile); } catch (IOException e) { LOGGER.error("Error while writing to " + urlFile, e); diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index e0874cd2..c8efdb32 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -74,7 +74,7 @@ public abstract class AlbumRipper extends AbstractRipper { Path urlFile = Paths.get(this.workingDir + "/urls.txt"); String text = url.toExternalForm() + System.lineSeparator(); try { - Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); itemsCompleted.put(url, urlFile); } catch (IOException e) { LOGGER.error("Error while writing to " + urlFile, e); From 3545cfbda9b8358352b1deb834108fe35508d3a1 Mon Sep 17 00:00:00 2001 From: Zsombor Date: Wed, 12 Oct 2022 23:17:28 +0200 Subject: [PATCH 347/512] Refactor code, to not repeat the same caching, getFirstPage implementation --- .../ripme/ripper/AbstractHTMLRipper.java | 5 ++++- .../ripme/ripper/rippers/AerisdiesRipper.java | 5 ----- .../ripper/rippers/AllporncomicRipper.java | 6 ------ .../ripme/ripper/rippers/BatoRipper.java | 6 ------ .../ripme/ripper/rippers/BcfakesRipper.java | 5 ----- .../rippers/BlackbrickroadofozRipper.java | 6 ------ .../ripme/ripper/rippers/ChanRipper.java | 7 ++----- .../ripme/ripper/rippers/CyberdropRipper.java | 5 ----- .../ripme/ripper/rippers/DribbbleRipper.java | 4 ---- .../ripper/rippers/DuckmoviesRipper.java | 5 ----- .../ripper/rippers/DynastyscansRipper.java | 6 ------ .../ripme/ripper/rippers/ErofusRipper.java | 5 ----- .../ripper/rippers/FemjoyhunterRipper.java | 6 ------ .../ripper/rippers/FitnakedgirlsRipper.java | 5 ----- .../ripme/ripper/rippers/FlickrRipper.java | 4 ---- .../ripme/ripper/rippers/FooktubeRipper.java | 4 ---- .../ripper/rippers/FreeComicOnlineRipper.java | 6 ------ .../ripper/rippers/GfycatporntubeRipper.java | 6 ------ .../ripper/rippers/GirlsOfDesireRipper.java | 5 ----- .../ripper/rippers/HentaidudeRipper.java | 6 ------ .../ripme/ripper/rippers/HentaifoxRipper.java | 6 ------ .../ripper/rippers/HentaiimageRipper.java | 7 ------- .../ripme/ripper/rippers/HqpornerRipper.java | 3 +-- .../ripme/ripper/rippers/HypnohubRipper.java | 6 ------ .../ripme/ripper/rippers/ImagebamRipper.java | 5 ----- .../ripper/rippers/ImagevenueRipper.java | 5 ----- .../ripme/ripper/rippers/ImgboxRipper.java | 4 ---- .../ripper/rippers/JabArchivesRipper.java | 6 ------ .../ripper/rippers/JagodibujaRipper.java | 6 ------ .../ripme/ripper/rippers/KingcomixRipper.java | 7 ------- .../ripme/ripper/rippers/LusciousRipper.java | 5 +---- .../ripme/ripper/rippers/ManganeloRipper.java | 6 ------ .../ripme/ripper/rippers/MeituriRipper.java | 5 ----- .../ripme/ripper/rippers/ModelxRipper.java | 5 ----- .../ripper/rippers/MyhentaicomicsRipper.java | 3 +-- .../ripper/rippers/MyhentaigalleryRipper.java | 6 ------ .../ripper/rippers/MyreadingmangaRipper.java | 6 ------ .../ripme/ripper/rippers/NatalieMuRipper.java | 5 ----- .../ripme/ripper/rippers/NfsfwRipper.java | 21 ++++++++----------- .../ripme/ripper/rippers/NudeGalsRipper.java | 12 +---------- .../ripme/ripper/rippers/OglafRipper.java | 6 ------ .../ripme/ripper/rippers/PichunterRipper.java | 6 ------ .../ripme/ripper/rippers/PicstatioRipper.java | 6 ------ .../ripme/ripper/rippers/PorncomixRipper.java | 6 ------ .../ripper/rippers/PorncomixinfoRipper.java | 6 ------ .../ripme/ripper/rippers/PornpicsRipper.java | 6 ------ .../ripme/ripper/rippers/RulePornRipper.java | 5 ----- .../ripper/rippers/ShesFreakyRipper.java | 6 ------ .../ripme/ripper/rippers/SinfestRipper.java | 6 ------ .../ripme/ripper/rippers/SmuttyRipper.java | 5 ----- .../ripme/ripper/rippers/SoundgasmRipper.java | 2 +- .../ripme/ripper/rippers/SpankbangRipper.java | 5 ----- .../ripme/ripper/rippers/StaRipper.java | 6 ------ .../ripme/ripper/rippers/TapasticRipper.java | 5 ----- .../ripper/rippers/TeenplanetRipper.java | 5 ----- .../ripme/ripper/rippers/ThechiveRipper.java | 6 ------ .../ripper/rippers/TheyiffgalleryRipper.java | 6 ------ .../ripme/ripper/rippers/VidbleRipper.java | 5 ----- .../ripme/ripper/rippers/ViewcomicRipper.java | 6 ------ .../ripme/ripper/rippers/VscoRipper.java | 5 ----- .../ripper/rippers/WordpressComicRipper.java | 6 ------ .../ripme/ripper/rippers/XcartxRipper.java | 5 ----- .../ripme/ripper/rippers/XhamsterRipper.java | 12 +++++------ .../ripme/ripper/rippers/XvideosRipper.java | 5 ----- .../ripme/ripper/rippers/YoupornRipper.java | 5 ----- .../ripme/ripper/rippers/YuvutuRipper.java | 4 ---- .../ripme/ripper/rippers/tamindirmp3.java | 6 ------ 67 files changed, 25 insertions(+), 363 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 856d32f5..a5d488a9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -23,6 +23,7 @@ import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; import com.rarchives.ripme.ui.MainWindow; import com.rarchives.ripme.ui.RipStatusMessage; +import com.rarchives.ripme.utils.Http; /** * Simplified ripper, designed for ripping from sites by parsing HTML. @@ -41,7 +42,9 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { protected abstract String getDomain(); public abstract String getHost(); - protected abstract Document getFirstPage() throws IOException; + protected Document getFirstPage() throws IOException { + return Http.url(url).get(); + } protected Document getCachedFirstPage() throws IOException { if (cachedFirstPage == null) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/AerisdiesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/AerisdiesRipper.java index f14543e9..c68d9784 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/AerisdiesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/AerisdiesRipper.java @@ -62,11 +62,6 @@ public class AerisdiesRipper extends AbstractHTMLRipper { return super.getAlbumTitle(url); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document page) { List imageURLs = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/AllporncomicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/AllporncomicRipper.java index a5fbbd0f..f0a31ed1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/AllporncomicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/AllporncomicRipper.java @@ -46,12 +46,6 @@ public class AllporncomicRipper extends AbstractHTMLRipper { "allporncomic.com/TITLE/CHAPTER - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java index 086e2be7..2d3194c6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java @@ -13,7 +13,6 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; public class BatoRipper extends AbstractHTMLRipper { @@ -94,11 +93,6 @@ public class BatoRipper extends AbstractHTMLRipper { return m.matches(); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } @Override public List getURLsFromPage(Document doc) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/BcfakesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/BcfakesRipper.java index 2a77f02d..2798b1ea 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/BcfakesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/BcfakesRipper.java @@ -47,11 +47,6 @@ public class BcfakesRipper extends AbstractHTMLRipper { + " Got: " + url); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - @Override public Document getNextPage(Document doc) throws IOException { // Find next page diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java index cb5d4b14..d99fe61d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/BlackbrickroadofozRipper.java @@ -41,12 +41,6 @@ public class BlackbrickroadofozRipper extends AbstractHTMLRipper { "www.blackbrickroadofoz.com/comic/PAGE - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public Document getNextPage(Document doc) throws IOException { sleep(1000); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java index 601bacae..875c0849 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java @@ -2,7 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.rippers.ripperhelpers.ChanSite; -import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.Utils; import com.rarchives.ripme.utils.RipUtils; import java.io.IOException; import java.net.MalformedURLException; @@ -13,7 +13,6 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.rarchives.ripme.utils.Utils; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -195,11 +194,9 @@ public class ChanRipper extends AbstractHTMLRipper { return this.url.getHost(); } - @Override public Document getFirstPage() throws IOException { - return Http.url(this.url).get(); + return super.getFirstPage(); } - private boolean isURLBlacklisted(String url) { for (String blacklist_item : url_piece_blacklist) { if (url.contains(blacklist_item)) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java index f288592a..0e0220f9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java @@ -23,11 +23,6 @@ public class CyberdropRipper extends AbstractHTMLRipper { return "cyberdrop"; } - @Override - protected Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - @Override public String getDomain() { return "cyberdrop.me"; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DribbbleRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DribbbleRipper.java index dc8cd77e..c463f5a8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DribbbleRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DribbbleRipper.java @@ -41,10 +41,6 @@ public class DribbbleRipper extends AbstractHTMLRipper { "dribbble.com/albumid - got " + url + "instead"); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } @Override public Document getNextPage(Document doc) throws IOException { // Find next page diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DuckmoviesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DuckmoviesRipper.java index 696ea015..e07cb683 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DuckmoviesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DuckmoviesRipper.java @@ -80,11 +80,6 @@ public class DuckmoviesRipper extends AbstractSingleFileRipper { return explicit_domains.contains(url_name.split("/")[2]); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(this.url).get(); - } - @Override public List getURLsFromPage(Document doc) { List results = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DynastyscansRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DynastyscansRipper.java index 37d3ad93..f8eaa72d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DynastyscansRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DynastyscansRipper.java @@ -42,12 +42,6 @@ public class DynastyscansRipper extends AbstractHTMLRipper { "dynasty-scans.com/chapters/ID - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public Document getNextPage(Document doc) throws IOException { Element elem = doc.select("a[id=next_link]").first(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java index dc535dea..cd95d7fc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java @@ -48,11 +48,6 @@ public class ErofusRipper extends AbstractHTMLRipper { return m.group(m.groupCount()); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document page) { LOGGER.info(page); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FemjoyhunterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FemjoyhunterRipper.java index 1922002b..2d1a0eb9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FemjoyhunterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FemjoyhunterRipper.java @@ -41,12 +41,6 @@ public class FemjoyhunterRipper extends AbstractHTMLRipper { "femjoyhunter.com/ID - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FitnakedgirlsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FitnakedgirlsRipper.java index de6fb73d..19afdf3c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FitnakedgirlsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FitnakedgirlsRipper.java @@ -46,11 +46,6 @@ public class FitnakedgirlsRipper extends AbstractHTMLRipper { "Expected fitnakedgirls.com gallery format: " + "fitnakedgirls.com/gallery/####" + " Got: " + url); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java index c7d91160..901a857c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java @@ -227,10 +227,6 @@ public class FlickrRipper extends AbstractHTMLRipper { + " Got: " + url); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } @Override public List getURLsFromPage(Document doc) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FooktubeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FooktubeRipper.java index 3cda70b2..180dc20c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FooktubeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FooktubeRipper.java @@ -34,10 +34,6 @@ public class FooktubeRipper extends AbstractSingleFileRipper { return "mulemax.com"; } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } @Override public boolean canRip(URL url) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FreeComicOnlineRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FreeComicOnlineRipper.java index e08d77fd..a39d3b9b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FreeComicOnlineRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FreeComicOnlineRipper.java @@ -44,12 +44,6 @@ public class FreeComicOnlineRipper extends AbstractHTMLRipper { "freecomiconline.me/TITLE/CHAPTER - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public Document getNextPage(Document doc) throws IOException { String nextPage = doc.select("div.select-pagination a").get(1).attr("href"); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java index fd8c292a..73669104 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java @@ -40,12 +40,6 @@ public class GfycatporntubeRipper extends AbstractSingleFileRipper { "gfycatporntube.com/NAME - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java index 6a269dca..664828f7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java @@ -60,11 +60,6 @@ public class GirlsOfDesireRipper extends AbstractHTMLRipper { + " Got: " + url); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java index 2c7ae568..25bc57d3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java @@ -50,12 +50,6 @@ public class HentaidudeRipper extends AbstractSingleFileRipper { "Expected hqporner URL format: " + "hentaidude.com/VIDEO - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java index a811cf9d..a6652b41 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java @@ -41,12 +41,6 @@ public class HentaifoxRipper extends AbstractHTMLRipper { "https://hentaifox.com/gallery/ID - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { LOGGER.info(doc); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiimageRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiimageRipper.java index df7bfb96..45628e82 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiimageRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiimageRipper.java @@ -52,13 +52,6 @@ public class HentaiimageRipper extends AbstractHTMLRipper { "https://hentai-image.com/image/ID - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - - @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java index 17c379e3..c8d383f5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java @@ -64,8 +64,7 @@ public class HqpornerRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); + return super.getFirstPage(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HypnohubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HypnohubRipper.java index 5b481258..15420655 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HypnohubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HypnohubRipper.java @@ -46,12 +46,6 @@ public class HypnohubRipper extends AbstractHTMLRipper { "hypnohub.net/pool/show/ID - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - private String ripPost(String url) throws IOException { LOGGER.info(url); Document doc = Http.url(url).get(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index 6aeb564f..35e8d0b5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -56,11 +56,6 @@ public class ImagebamRipper extends AbstractHTMLRipper { + " Got: " + url); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - @Override public Document getNextPage(Document doc) throws IOException { // Find next page diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java index 8e250ae9..cc58c561 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java @@ -56,11 +56,6 @@ public class ImagevenueRipper extends AbstractHTMLRipper { + " Got: " + url); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList<>(); for (Element thumb : doc.select("a[target=_blank]")) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgboxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgboxRipper.java index f3050a13..b32fcad4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgboxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgboxRipper.java @@ -40,10 +40,6 @@ public class ImgboxRipper extends AbstractHTMLRipper { "imgbox.com/g/albumid - got " + url + "instead"); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } @Override public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/JabArchivesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/JabArchivesRipper.java index e7af19bc..84fad505 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/JabArchivesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/JabArchivesRipper.java @@ -55,12 +55,6 @@ public class JabArchivesRipper extends AbstractHTMLRipper { "jabarchives.com/main/view/albumname - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public Document getNextPage(Document doc) throws IOException { // Find next page diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/JagodibujaRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/JagodibujaRipper.java index d5df1fe5..2c16b2e5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/JagodibujaRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/JagodibujaRipper.java @@ -40,12 +40,6 @@ public class JagodibujaRipper extends AbstractHTMLRipper { throw new MalformedURLException("Expected jagodibuja.com gallery formats hwww.jagodibuja.com/Comic name/ got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/KingcomixRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/KingcomixRipper.java index 4876237e..bb8194bc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/KingcomixRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/KingcomixRipper.java @@ -41,13 +41,6 @@ public class KingcomixRipper extends AbstractHTMLRipper { "kingcomix.com/COMIX - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - - @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java index 09fd8247..277c5ff0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java @@ -37,10 +37,7 @@ public class LusciousRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - Document page = Http.url(url).get(); - LOGGER.info("First page is " + url); - return page; + return super.getFirstPage(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java index f4325aa1..c5f6b142 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java @@ -48,12 +48,6 @@ public class ManganeloRipper extends AbstractHTMLRipper { "/manganelo.com/manga/ID - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public Document getNextPage(Document doc) throws IOException { Element elem = doc.select("div.btn-navigation-chap > a.back").first(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MeituriRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MeituriRipper.java index d5c198bb..2c83ce7e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MeituriRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MeituriRipper.java @@ -49,11 +49,6 @@ public class MeituriRipper extends AbstractHTMLRipper { "Expected tujigu.com URL format: " + "tujigu.com/a/albumid/ - got " + url + "instead"); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ModelxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ModelxRipper.java index 0b513b37..c2d6ed47 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ModelxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ModelxRipper.java @@ -41,11 +41,6 @@ public class ModelxRipper extends AbstractHTMLRipper { throw new MalformedURLException("Expected URL format: http://www.modelx.org/[category (one or more)]/xxxxx got: " + url); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document page) { List result = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java index 453826a3..e206925f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java @@ -86,8 +86,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); + return super.getFirstPage(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaigalleryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaigalleryRipper.java index d8422942..c9f4c0bd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaigalleryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaigalleryRipper.java @@ -40,12 +40,6 @@ public class MyhentaigalleryRipper extends AbstractHTMLRipper { + "myhentaigallery.com/gallery/thumbnails/ID - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MyreadingmangaRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MyreadingmangaRipper.java index 20a3cf2d..30fab521 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MyreadingmangaRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MyreadingmangaRipper.java @@ -41,12 +41,6 @@ public class MyreadingmangaRipper extends AbstractHTMLRipper { + "myreadingmanga.info/title - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NatalieMuRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NatalieMuRipper.java index 952b434e..8cf24fd8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NatalieMuRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NatalieMuRipper.java @@ -79,11 +79,6 @@ public class NatalieMuRipper extends AbstractHTMLRipper { return this.url.getHost(); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(this.url).get(); - } - @Override public List getURLsFromPage(Document page) { List imageURLs = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java index d0769138..03a4717f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java @@ -29,8 +29,6 @@ public class NfsfwRipper extends AbstractHTMLRipper { "https?://[wm.]*nfsfw.com/gallery/v/[^/]+/(.+)$" ); - // cached first page - private Document fstPage; // threads pool for downloading images from image pages private DownloadThreadPool nfsfwThreadPool; @@ -49,13 +47,6 @@ public class NfsfwRipper extends AbstractHTMLRipper { return HOST; } - @Override - protected Document getFirstPage() throws IOException { - // cache the first page - this.fstPage = Http.url(url).get(); - return fstPage; - } - @Override public Document getNextPage(Document page) throws IOException { String nextURL = null; @@ -157,9 +148,15 @@ public class NfsfwRipper extends AbstractHTMLRipper { @Override public boolean pageContainsAlbums(URL url) { - List imageURLs = getImagePageURLs(fstPage); - List subalbumURLs = getSubalbumURLs(fstPage); - return imageURLs.isEmpty() && !subalbumURLs.isEmpty(); + try { + final var fstPage = getCachedFirstPage(); + List imageURLs = getImagePageURLs(fstPage); + List subalbumURLs = getSubalbumURLs(fstPage); + return imageURLs.isEmpty() && !subalbumURLs.isEmpty(); + } catch (IOException e) { + LOGGER.error("Unable to load " + url, e); + return false; + } } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NudeGalsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NudeGalsRipper.java index 3300da50..ea145aad 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NudeGalsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NudeGalsRipper.java @@ -16,8 +16,6 @@ import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; public class NudeGalsRipper extends AbstractHTMLRipper { - // Current HTML document - private Document albumDoc = null; public NudeGalsRipper(URL url) throws IOException { super(url); @@ -50,14 +48,6 @@ public class NudeGalsRipper extends AbstractHTMLRipper { + " Got: " + url); } - @Override - public Document getFirstPage() throws IOException { - if (albumDoc == null) { - albumDoc = Http.url(url).get(); - } - return albumDoc; - } - @Override public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList<>(); @@ -77,4 +67,4 @@ public class NudeGalsRipper extends AbstractHTMLRipper { // Send referrer when downloading images addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), null); } -} \ No newline at end of file +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/OglafRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/OglafRipper.java index a5183397..e03d3bdc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/OglafRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/OglafRipper.java @@ -46,12 +46,6 @@ public class OglafRipper extends AbstractHTMLRipper { return getDomain(); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public Document getNextPage(Document doc) throws IOException { if (doc.select("div#nav > a > div#nx").first() == null) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PichunterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PichunterRipper.java index e6c5d110..bdb5f528 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PichunterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PichunterRipper.java @@ -63,12 +63,6 @@ public class PichunterRipper extends AbstractHTMLRipper { return m.matches(); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public Document getNextPage(Document doc) throws IOException { // We use comic-nav-next to the find the next page diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PicstatioRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PicstatioRipper.java index 1bd103b5..65d43d39 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PicstatioRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PicstatioRipper.java @@ -51,12 +51,6 @@ public class PicstatioRipper extends AbstractHTMLRipper { "www.picstatio.com//ID - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public Document getNextPage(Document doc) throws IOException { if (doc.select("a.next_page") != null) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixRipper.java index b4579684..f021269f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixRipper.java @@ -41,12 +41,6 @@ public class PorncomixRipper extends AbstractHTMLRipper { "porncomix.info/comic - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixinfoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixinfoRipper.java index 241ad5d7..8aef59a6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixinfoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixinfoRipper.java @@ -41,12 +41,6 @@ public class PorncomixinfoRipper extends AbstractHTMLRipper { "porncomixinfo.net/chapter/CHAP/ID - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public Document getNextPage(Document doc) throws IOException { // Find next page diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PornpicsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PornpicsRipper.java index b779c480..799f7294 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PornpicsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PornpicsRipper.java @@ -41,12 +41,6 @@ public class PornpicsRipper extends AbstractHTMLRipper { "www.pornpics.com/galleries/ID - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RulePornRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RulePornRipper.java index c9c487a7..be33c945 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RulePornRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RulePornRipper.java @@ -40,11 +40,6 @@ public class RulePornRipper extends AbstractSingleFileRipper { "Expected ruleporn.com URL format: " + "ruleporn.com/NAME - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ShesFreakyRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ShesFreakyRipper.java index 73dad1b1..b96e2f6b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ShesFreakyRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ShesFreakyRipper.java @@ -12,7 +12,6 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; public class ShesFreakyRipper extends AbstractHTMLRipper { @@ -41,11 +40,6 @@ public class ShesFreakyRipper extends AbstractHTMLRipper { + "shesfreaky.com/gallery/... - got " + url + "instead"); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java index d6a0f9cb..f3a216f4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java @@ -41,12 +41,6 @@ public class SinfestRipper extends AbstractHTMLRipper { "sinfest.net/view.php?date=XXXX-XX-XX/ - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public Document getNextPage(Document doc) throws IOException { Element elem = doc.select("td.style5 > a > img").last(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java index b61f2fef..ad00e5c8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SmuttyRipper.java @@ -89,11 +89,6 @@ public class SmuttyRipper extends AbstractHTMLRipper { } @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index)); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java index 65a1f1de..884b5bc9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java @@ -43,7 +43,7 @@ public class SoundgasmRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - return Http.url(url).get(); + return super.getFirstPage(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SpankbangRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SpankbangRipper.java index bca5ef66..9ea1a130 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SpankbangRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SpankbangRipper.java @@ -28,11 +28,6 @@ public class SpankbangRipper extends AbstractSingleFileRipper { return "spankbang.com"; } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java index b331bbce..ad73e452 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java @@ -47,12 +47,6 @@ public class StaRipper extends AbstractHTMLRipper { "sta.sh/ALBUMID - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java index 369ce741..d4a0d8f3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java @@ -46,11 +46,6 @@ public class TapasticRipper extends AbstractHTMLRipper { return "tapas"; } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document page) { List urls = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TeenplanetRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TeenplanetRipper.java index 9791ab90..25edb5f7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TeenplanetRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TeenplanetRipper.java @@ -34,11 +34,6 @@ public class TeenplanetRipper extends AbstractHTMLRipper { return HOST; } - @Override - protected Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - @Override protected List getURLsFromPage(Document page) { List imageURLs = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ThechiveRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ThechiveRipper.java index 3c9d751d..8105fe73 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ThechiveRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ThechiveRipper.java @@ -70,12 +70,6 @@ public class ThechiveRipper extends AbstractHTMLRipper { + "thechive.com/YEAR/MONTH/DAY/POSTTITLE/ OR i.thechive.com/username, got " + url + " instead."); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { List result; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TheyiffgalleryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TheyiffgalleryRipper.java index ac3e363c..3f616faa 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TheyiffgalleryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TheyiffgalleryRipper.java @@ -41,12 +41,6 @@ public class TheyiffgalleryRipper extends AbstractHTMLRipper { "theyiffgallery.com/index?/category/#### - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public Document getNextPage(Document doc) throws IOException { String nextPage = doc.select("span.navPrevNext > a").attr("href"); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java index 9a962f3a..e3888f75 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java @@ -45,11 +45,6 @@ public class VidbleRipper extends AbstractHTMLRipper { + " Got: " + url); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { return getURLsFromPageStatic(doc); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java index 1cdd6b7d..fddbf1f2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java @@ -60,12 +60,6 @@ public class ViewcomicRipper extends AbstractHTMLRipper { "view-comic.com/COMIC_NAME - got " + url + " instead"); } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document doc) { List result = new ArrayList(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java index 8ff4c85f..8e3b9d15 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java @@ -215,11 +215,6 @@ public class VscoRipper extends AbstractHTMLRipper { return DOMAIN; } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - @Override public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index)); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index 0589f29d..1019c7d8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -421,10 +421,4 @@ public class WordpressComicRipper extends AbstractHTMLRipper { } - - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XcartxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XcartxRipper.java index 64829a0b..0b616726 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XcartxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XcartxRipper.java @@ -44,11 +44,6 @@ public class XcartxRipper extends AbstractHTMLRipper { } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } - @Override public List getURLsFromPage(Document page) { List imageURLs = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index d9707646..6da81338 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -106,13 +106,6 @@ public class XhamsterRipper extends AbstractHTMLRipper { return m.matches(); } - - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - @Override public boolean canRip(URL url) { Pattern p = Pattern.compile("^https?://([\\w\\w]*\\.)?xhamster([^<]*)\\.(com|one|desi)/photos/gallery/.*?(\\d+)$"); @@ -152,6 +145,11 @@ public class XhamsterRipper extends AbstractHTMLRipper { } + @Override + public Document getFirstPage() throws IOException { + return super.getFirstPage(); + } + @Override public List getURLsFromPage(Document doc) { LOGGER.debug("Checking for urls"); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java index 0fdef868..5bcb89cf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java @@ -24,11 +24,6 @@ public class XvideosRipper extends AbstractSingleFileRipper { super(url); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(this.url).get(); - } - @Override public String getHost() { return HOST; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/YoupornRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/YoupornRipper.java index e99ffef5..b3e5f4f0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/YoupornRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/YoupornRipper.java @@ -40,11 +40,6 @@ public class YoupornRipper extends AbstractSingleFileRipper { return m.matches(); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(this.url).get(); - } - @Override public List getURLsFromPage(Document doc) { List results = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/YuvutuRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/YuvutuRipper.java index 97365aa8..1fe6513f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/YuvutuRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/YuvutuRipper.java @@ -50,10 +50,6 @@ public class YuvutuRipper extends AbstractHTMLRipper { "yuvutu.com/modules.php?name=YuGallery&action=view&set_id=albumid - got " + url + "instead"); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - } @Override public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList<>(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/tamindirmp3.java b/src/main/java/com/rarchives/ripme/ripper/rippers/tamindirmp3.java index 2c82d849..d10e1205 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/tamindirmp3.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/tamindirmp3.java @@ -40,12 +40,6 @@ public class tamindirmp3 extends AbstractHTMLRipper { "tamindir.com/files/albumid - got " + url + "instead"); } - @Override - public Document getFirstPage() throws IOException { - return Http.url(url).get(); - - } - @Override public List getURLsFromPage(Document doc) { List music = new ArrayList<>(); From 76d1905807286574bc319f9bf8855ef0660532c3 Mon Sep 17 00:00:00 2001 From: noesterle Date: Thu, 20 Oct 2022 23:46:43 -0400 Subject: [PATCH 348/512] Fixed relative path issue. Relative path was being made from subfolder to working dir, now goes from working dir to subfolder. --- src/main/java/com/rarchives/ripme/utils/Utils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index e15dd9cc..e24677ef 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -323,7 +323,7 @@ public class Utils { */ public static String removeCWD(Path saveAs) { try { - return saveAs.relativize(Paths.get(".").toAbsolutePath()).toString(); + return Paths.get(".").toAbsolutePath().relativize(saveAs).toString(); } catch (IllegalArgumentException e) { return saveAs.toString(); From dd307bc4c8e66bbddf5136995bff4bc9f0bcbb8c Mon Sep 17 00:00:00 2001 From: noesterle Date: Thu, 20 Oct 2022 23:47:51 -0400 Subject: [PATCH 349/512] Saved name of image file to be a part of saveAs, allowing images to be put in the subfolder with the same filename. --- .../java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index 10c633ba..35fb9756 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -210,7 +210,7 @@ public class ImgurRipper extends AlbumRipper { if (Utils.getConfigBoolean("download.save_order", true)) { saveAs.resolve(String.format("%03d_", index)); } - saveAs.resolve(imgurImage.getSaveAs().replaceAll("\\?\\d", "")); + saveAs = saveAs.resolve(imgurImage.getSaveAs().replaceAll("\\?\\d", "")); addURLToDownload(imgurImage.url, saveAs); } } From c9c46d6ae3295cd4ce86ff982f4f891e86e12a54 Mon Sep 17 00:00:00 2001 From: noesterle Date: Sat, 22 Oct 2022 18:00:42 -0400 Subject: [PATCH 350/512] Added ability to sanatize reddit titles saved as files. --- .../com/rarchives/ripme/ripper/rippers/RedditRipper.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index f03511a1..12a11b6e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -292,7 +292,7 @@ public class RedditRipper extends AlbumRipper { try { saveFileAs = Utils.getPath(workingDir + "/" - + id + "_" + title + + id + "_" + Utils.filesystemSafe(title) + ".html"); OutputStream out = Files.newOutputStream(saveFileAs); out.write(html.getBytes()); @@ -411,12 +411,12 @@ public class RedditRipper extends AlbumRipper { if (m.matches()) { // It's from reddituploads. Assume .jpg extension. String savePath = this.workingDir + "/"; - savePath += id + "-" + m.group(1) + title + ".jpg"; + savePath += id + "-" + m.group(1) + Utils.filesystemSafe(title) + ".jpg"; addURLToDownload(urls.get(0), Utils.getPath(savePath)); } if (url.contains("v.redd.it")) { String savePath = this.workingDir + "/"; - savePath += id + "-" + url.split("/")[3] + title + ".mp4"; + savePath += id + "-" + url.split("/")[3] + Utils.filesystemSafe(title) + ".mp4"; URL urlToDownload = parseRedditVideoMPD(urls.get(0).toExternalForm()); if (urlToDownload != null) { LOGGER.info("url: " + urlToDownload + " file: " + savePath); @@ -424,7 +424,7 @@ public class RedditRipper extends AlbumRipper { } } else { - addURLToDownload(urls.get(0), id + title, "", theUrl, null); + addURLToDownload(urls.get(0), Utils.filesystemSafe(id + title), "", theUrl, null); } } else if (urls.size() > 1) { for (int i = 0; i < urls.size(); i++) { From edf320a7cbd2acc8060bcbaa917b4827027b50f5 Mon Sep 17 00:00:00 2001 From: brantspar <125783125+brantspar@users.noreply.github.com> Date: Tue, 28 Feb 2023 21:50:36 +1100 Subject: [PATCH 351/512] * Additional logging in AbstractHTMLRipper * fixed ImagefapRipper.java to handle new URL schema + possible IOError --- .../ripme/ripper/AbstractHTMLRipper.java | 9 ++ .../ripme/ripper/rippers/ImagefapRipper.java | 102 ++++++++++++------ 2 files changed, 76 insertions(+), 35 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index a5d488a9..1a5d2bb4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -124,10 +124,16 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { } List doclocation = new ArrayList<>(); + + LOGGER.info("Got doc location " + doc.location()); + while (doc != null) { + LOGGER.info("Processing a doc..."); + // catch if we saw a doc location already, save the ones seen in a list if (doclocation.contains(doc.location())) { + LOGGER.info("Already processed location " + doc.location() + " breaking"); break; } doclocation.add(doc.location()); @@ -136,6 +142,9 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { sendUpdate(STATUS.DOWNLOAD_COMPLETE_HISTORY, "Already seen the last " + alreadyDownloadedUrls + " images ending rip"); break; } + + LOGGER.info("retrieving urls from doc"); + List imageURLs = getURLsFromPage(doc); // If hasASAPRipping() returns true then the ripper will handle downloading the files // if not it's done in the following block of code diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java index 87dc18a5..228ae65a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java @@ -17,12 +17,11 @@ import com.rarchives.ripme.utils.Http; public class ImagefapRipper extends AbstractHTMLRipper { - private boolean isNewAlbumType = false; - private int callsMade = 0; private long startTime = System.nanoTime(); private static final int RETRY_LIMIT = 10; + private static final int HTTP_RETRY_LIMIT = 3; private static final int RATE_LIMIT_HOUR = 1000; // All sleep times are in milliseconds @@ -50,11 +49,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { @Override public URL sanitizeURL(URL url) throws MalformedURLException { String gid = getGID(url); - String newURL = "https://www.imagefap.com/gallery.php?"; - if (isNewAlbumType) { - newURL += "p"; - } - newURL += "gid=" + gid + "&view=2"; + String newURL = "https://www.imagefap.com/pictures/" + gid + "/random-string"; LOGGER.debug("Changed URL from " + url + " to " + newURL); return new URL(newURL); } @@ -63,39 +58,29 @@ public class ImagefapRipper extends AbstractHTMLRipper { public String getGID(URL url) throws MalformedURLException { Pattern p; Matcher m; + // Old format (I suspect no longer supported) p = Pattern.compile("^.*imagefap.com/gallery.php\\?pgid=([a-f0-9]+).*$"); m = p.matcher(url.toExternalForm()); if (m.matches()) { - isNewAlbumType = true; return m.group(1); } + p = Pattern.compile("^.*imagefap.com/gallery.php\\?gid=([0-9]+).*$"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); } - p = Pattern.compile("^.*imagefap.com/pictures/([0-9]+).*$"); - m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - p = Pattern.compile("^.*imagefap.com/pictures/([a-f0-9]+).*$"); - m = p.matcher(url.toExternalForm()); - if (m.matches()) { - isNewAlbumType = true; - return m.group(1); - } - - p = Pattern.compile("^.*imagefap.com/gallery/([0-9]+).*$"); - m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } p = Pattern.compile("^.*imagefap.com/gallery/([a-f0-9]+).*$"); m = p.matcher(url.toExternalForm()); if (m.matches()) { - isNewAlbumType = true; + return m.group(1); + } + + // most recent format + p = Pattern.compile("^.*imagefap.com/pictures/([a-f0-9]+).*$"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { return m.group(1); } @@ -108,7 +93,12 @@ public class ImagefapRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - return getPageWithRetries(url); + + Document firstPage = getPageWithRetries(url); + + sendUpdate(STATUS.LOADING_RESOURCE, "Loading first page..."); + + return firstPage; } @Override @@ -116,7 +106,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { String nextURL = null; for (Element a : doc.select("a.link3")) { if (a.text().contains("next")) { - nextURL = "https://imagefap.com/gallery.php" + a.attr("href"); + nextURL = this.sanitizeURL(this.url) + a.attr("href"); break; } } @@ -125,6 +115,9 @@ public class ImagefapRipper extends AbstractHTMLRipper { } // Sleep before fetching next page. sleep(PAGE_SLEEP_TIME); + + sendUpdate(STATUS.LOADING_RESOURCE, "Loading next page URL: " + nextURL); + LOGGER.info("Attempting to load next page URL: " + nextURL); // Load next page Document nextPage = getPageWithRetries(new URL(nextURL)); @@ -134,17 +127,27 @@ public class ImagefapRipper extends AbstractHTMLRipper { @Override public List getURLsFromPage(Document doc) { + List imageURLs = new ArrayList<>(); + + LOGGER.debug("Trying to get URLs from document... "); + for (Element thumb : doc.select("#gallery img")) { if (!thumb.hasAttr("src") || !thumb.hasAttr("width")) { continue; } String image = getFullSizedImage("https://www.imagefap.com" + thumb.parent().attr("href")); + + if(image == null) + throw new RuntimeException("Unable to extract image URL from single image page! Unable to continue"); + imageURLs.add(image); if (isThisATest()) { break; } } + LOGGER.debug("Adding " + imageURLs.size() + " URLs to download"); + return imageURLs; } @@ -176,6 +179,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { Document doc = getPageWithRetries(new URL(pageURL)); return doc.select("img#mainPhoto").attr("src"); } catch (IOException e) { + LOGGER.debug("Unable to get full size image URL from page URL " + pageURL + " because: " + e.getMessage()); return null; } } @@ -187,9 +191,10 @@ public class ImagefapRipper extends AbstractHTMLRipper { * @throws IOException If page loading errors, or if retries are exhausted */ private Document getPageWithRetries(URL url) throws IOException { - Document doc; + Document doc = null; int retries = RETRY_LIMIT; while (true) { + sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm()); // For debugging rate limit checker. Useful to track wheter the timeout should be altered or not. @@ -197,15 +202,42 @@ public class ImagefapRipper extends AbstractHTMLRipper { checkRateLimit(); LOGGER.info("Retrieving " + url); - doc = Http.url(url) - .get(); + + boolean httpCallThrottled = false; + int httpAttempts = 0; + // we attempt the http call, knowing it can fail for network reasons + while(true) { + httpAttempts++; + try { + doc = Http.url(url).get(); + } catch(IOException e) { - if (doc.toString().contains("Your IP made too many requests to our servers and we need to check that you are a real human being")) { + LOGGER.info("Retrieving " + url + " error: " + e.getMessage()); + + if(e.getMessage().contains("404")) + throw new IOException("Gallery/Page not found!"); + + if(httpAttempts < HTTP_RETRY_LIMIT) { + sendUpdate(STATUS.DOWNLOAD_WARN, "HTTP call failed: " + e.getMessage() + " retrying " + httpAttempts + " / " + HTTP_RETRY_LIMIT); + + // we sleep for a few seconds + sleep(PAGE_SLEEP_TIME); + continue; + } else { + sendUpdate(STATUS.DOWNLOAD_WARN, "HTTP call failed too many times: " + e.getMessage() + " treating this as a throttle"); + httpCallThrottled = true; + } + } + // no errors, we exit + break; + } + + if (httpCallThrottled || (doc != null && doc.toString().contains("Your IP made too many requests to our servers and we need to check that you are a real human being"))) { if (retries == 0) { throw new IOException("Hit rate limit and maximum number of retries, giving up"); } - String message = "Hit rate limit while loading " + url + ", sleeping for " + IP_BLOCK_SLEEP_TIME + "ms, " + retries + " retries remaining"; + String message = "Probably hit rate limit while loading " + url + ", sleeping for " + IP_BLOCK_SLEEP_TIME + "ms, " + retries + " retries remaining"; LOGGER.warn(message); sendUpdate(STATUS.DOWNLOAD_WARN, message); retries--; @@ -214,8 +246,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { } catch (InterruptedException e) { throw new IOException("Interrupted while waiting for rate limit to subside"); } - } - else { + } else { return doc; } } @@ -245,4 +276,5 @@ public class ImagefapRipper extends AbstractHTMLRipper { return duration; } + } From 830e22e1557e285ec44015b1842d4e6a5650ad06 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 11 Jun 2023 00:50:27 +0200 Subject: [PATCH 352/512] fix download link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 93e2c748..9887f724 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ RipMe is an album ripper for various websites. It is a cross-platform tool that ## Downloads -Download `ripme.jar` from the [latest release](/releases). For information about running the `.jar` file, see +Download `ripme.jar` from the [latest release](https://github.com/ripmeapp2/ripme/releases). For information about running the `.jar` file, see [the How To Run wiki](https://github.com/ripmeapp/ripme/wiki/How-To-Run-RipMe). The version number like ripme-1.7.94-17-2167aa34-feature_auto_release.jar contains a release number (1.7.94), given by From b109a34de9a2fc915aeb5b063717acd6db634705 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 11 Jun 2023 02:30:11 +0200 Subject: [PATCH 353/512] update versions, make it work with java-20 --- build.gradle.kts | 24 +++++++++++-------- .../ripper/rippers/FuraffinityRipper.java | 4 ++-- .../ripme/ripper/rippers/ImgurRipper.java | 6 ++--- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 9e441b35..ecaf37c0 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -17,19 +17,19 @@ repositories { dependencies { implementation("com.lmax:disruptor:3.4.4") - implementation("org.java-websocket:Java-WebSocket:1.5.2") - implementation("org.jsoup:jsoup:1.14.3") + implementation("org.java-websocket:Java-WebSocket:1.5.3") + implementation("org.jsoup:jsoup:1.16.1") implementation("org.json:json:20211205") - implementation("com.j2html:j2html:1.5.0") + implementation("com.j2html:j2html:1.6.0") implementation("commons-configuration:commons-configuration:1.10") implementation("commons-cli:commons-cli:1.5.0") - implementation("commons-io:commons-io:2.11.0") - implementation("org.apache.httpcomponents:httpclient:4.5.13") - implementation("org.apache.httpcomponents:httpmime:4.5.13") - implementation("org.apache.logging.log4j:log4j-api:2.17.1") - implementation("org.apache.logging.log4j:log4j-core:2.17.1") - implementation("org.graalvm.js:js:22.0.0.2") - testImplementation(enforcedPlatform("org.junit:junit-bom:5.8.2")) + implementation("commons-io:commons-io:2.13.0") + implementation("org.apache.httpcomponents:httpclient:4.5.14") + implementation("org.apache.httpcomponents:httpmime:4.5.14") + implementation("org.apache.logging.log4j:log4j-api:2.20.0") + implementation("org.apache.logging.log4j:log4j-core:2.20.0") + implementation("org.graalvm.js:js:22.3.2") + testImplementation(enforcedPlatform("org.junit:junit-bom:5.9.3")) testImplementation("org.junit.jupiter:junit-jupiter") } @@ -37,6 +37,10 @@ group = "com.rarchives.ripme" version = "1.7.94" description = "ripme" +jacoco { + toolVersion = "0.8.10" +} + jgitver { gitCommitIDLength = 8 nonQualifierBranches = "main,master" diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java index d3357e9e..dbb46fe1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java @@ -20,7 +20,7 @@ import org.jsoup.Connection.Response; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.safety.Whitelist; +import org.jsoup.safety.Safelist; import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; @@ -167,7 +167,7 @@ public class FuraffinityRipper extends AbstractHTMLRipper { ele.select("br").append("\\n"); ele.select("p").prepend("\\n\\n"); LOGGER.debug("Returning description at " + page); - String tempPage = Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false)); + String tempPage = Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Safelist.none(), new Document.OutputSettings().prettyPrint(false)); return documentz.select("meta[property=og:title]").attr("content") + "\n" + tempPage; // Overridden saveText takes first line and makes it the file name. } catch (IOException ioe) { LOGGER.info("Failed to get description " + page + " : '" + ioe.getMessage() + "'"); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index 35fb9756..c854ae8c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -16,7 +16,7 @@ import org.json.JSONObject; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.safety.Whitelist; +import org.jsoup.safety.Safelist; import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AlbumRipper; @@ -256,10 +256,10 @@ public class ImgurRipper extends AlbumRipper { LOGGER.info(" Retrieving " + strUrl); Document doc = getAlbumData("https://api.imgur.com/3/album/" + strUrl.split("/a/")[1]); // Try to use embedded JSON to retrieve images - LOGGER.info(Jsoup.clean(doc.body().toString(), Whitelist.none())); + LOGGER.info(Jsoup.clean(doc.body().toString(), Safelist.none())); try { - JSONObject json = new JSONObject(Jsoup.clean(doc.body().toString(), Whitelist.none())); + JSONObject json = new JSONObject(Jsoup.clean(doc.body().toString(), Safelist.none())); JSONArray jsonImages = json.getJSONObject("data").getJSONArray("images"); return createImgurAlbumFromJsonArray(url, jsonImages); } catch (JSONException e) { From 1898dd264b50fe48ba9fd5bea71316c54976dccb Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 11 Jun 2023 02:52:09 +0200 Subject: [PATCH 354/512] couple of tests flaky ArtstnRipperTest, BaraagRipperTest,CheveretoRipperTest, MastodonXyzRipperTest, MyhentaigalleryRipperTest, PawooRipperTest, SinfestRipperTest, HbrowseRipper flaky. --- .../rarchives/ripme/tst/ripper/rippers/ArtstnRipperTest.java | 2 ++ .../rarchives/ripme/tst/ripper/rippers/BaraagRipperTest.java | 2 ++ .../ripme/tst/ripper/rippers/CheveretoRipperTest.java | 1 + .../rarchives/ripme/tst/ripper/rippers/HbrowseRipperTest.java | 4 +++- .../ripme/tst/ripper/rippers/MastodonXyzRipperTest.java | 2 ++ .../ripme/tst/ripper/rippers/MyhentaigalleryRipperTest.java | 2 ++ .../rarchives/ripme/tst/ripper/rippers/PawooRipperTest.java | 2 ++ .../rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java | 2 ++ 8 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtstnRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtstnRipperTest.java index 7ce919da..8817b369 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtstnRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtstnRipperTest.java @@ -6,10 +6,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.ArtstnRipper; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class ArtstnRipperTest extends RippersTest { @Test + @Tag("flaky") public void testSingleProject() throws IOException { URL url = new URL("https://artstn.co/p/JlE15Z"); testRipper(new ArtstnRipper(url)); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BaraagRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BaraagRipperTest.java index 57105a9a..5e71de70 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BaraagRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BaraagRipperTest.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.BaraagRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class BaraagRipperTest extends RippersTest { @Test + @Tag("flaky") public void testRip() throws IOException { BaraagRipper ripper = new BaraagRipper(new URL("https://baraag.net/@darkshadow777/media")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java index 420fcb00..5b233431 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java @@ -9,6 +9,7 @@ import org.junit.jupiter.api.Test; public class CheveretoRipperTest extends RippersTest { @Test + @Tag("flaky") public void testTagFox() throws IOException { CheveretoRipper ripper = new CheveretoRipper(new URL("http://tag-fox.com/album/Thjb")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HbrowseRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HbrowseRipperTest.java index 35b8ffa6..e4b90c65 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HbrowseRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HbrowseRipperTest.java @@ -4,12 +4,14 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.HbrowseRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class HbrowseRipperTest extends RippersTest { @Test + @Tag("flaky") public void testPahealRipper() throws IOException { HbrowseRipper ripper = new HbrowseRipper(new URL("https://www.hbrowse.com/21013/c00001")); testRipper(ripper); } -} \ No newline at end of file +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MastodonXyzRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MastodonXyzRipperTest.java index 2f500d6a..adbd09c0 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MastodonXyzRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MastodonXyzRipperTest.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.MastodonXyzRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class MastodonXyzRipperTest extends RippersTest { @Test + @Tag("flaky") public void testRip() throws IOException { MastodonXyzRipper ripper = new MastodonXyzRipper(new URL("https://mastodon.xyz/@artwo/media")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaigalleryRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaigalleryRipperTest.java index 19f29945..54cc9bb2 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaigalleryRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaigalleryRipperTest.java @@ -5,10 +5,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.MyhentaigalleryRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class MyhentaigalleryRipperTest extends RippersTest { @Test + @Tag("flaky") public void testMyhentaigalleryAlbum() throws IOException { MyhentaigalleryRipper ripper = new MyhentaigalleryRipper( new URL("https://myhentaigallery.com/gallery/thumbnails/9201")); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PawooRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PawooRipperTest.java index 730a965c..f2f26b49 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PawooRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PawooRipperTest.java @@ -4,10 +4,12 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.PawooRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class PawooRipperTest extends RippersTest { @Test + @Tag("flaky") public void testRip() throws IOException { PawooRipper ripper = new PawooRipper(new URL("https://pawoo.net/@halki/media")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java index 2dd311f3..905034c8 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java @@ -5,10 +5,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.SinfestRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class SinfestRipperTest extends RippersTest { @Test + @Tag("flaky") public void testRip() throws IOException { SinfestRipper ripper = new SinfestRipper(new URL("http://sinfest.net/view.php?date=2000-01-17")); testRipper(ripper); From 1b83dc68aea66bce3627a05b2fa2ab568044e847 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 11 Jun 2023 03:11:18 +0200 Subject: [PATCH 355/512] update to gradle-8.1.1 --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ae04661e..fae08049 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 360ad950cf65644783ff4809fb162bf8b59f0b5f Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 11 Jun 2023 08:32:17 +0200 Subject: [PATCH 356/512] release 2.1.4 --- ripme.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ripme.json b/ripme.json index 0117f495..eef798d0 100644 --- a/ripme.json +++ b/ripme.json @@ -1,7 +1,8 @@ { - "latestVersion": "2.1.2-23-e5438e85", - "currentHash": "a2309ba7ec0ffe2abca4932bd1b291cf88d0e495c233b888aa8f42c459da2ee0", + "latestVersion": "2.1.3-15-1b83dc68", + "currentHash": "1b83dc68aea66bce3627a05b2fa2ab568044e847", "changeList": [ + "2.1.3-15-1b83dc68: relative path now from working dir to subfolder, allowing images to be put in subfolder with same filename, sanatize reddit titles saved as files, additional logging in AbstractHTMLRipper.", "2.1.2-23-e5438e85: caching of first page, retry sleep time, nhentai fixed", "2.1.2-3-ea90b172: better sanitize filenames for windows, save config on update value. reddit, print exceptions in loops and continue.", "2.1.1-3-536339dd: java-11+ necessary to run, work around non existing working directory.", From 8ba17b47917103200dde7f66fb1c713dba3473ca Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 11 Jun 2023 22:29:07 +0200 Subject: [PATCH 357/512] space in debug message for lastestversion --- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 7df775d9..4d2c7cac 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -188,7 +188,7 @@ public class UpdateUtils { for (int i = 0; i < oldVersions.length; i++) { if (newVersions[i] > oldVersions[i]) { - logger.debug("oldVersion " + getThisJarVersion() + " < latestVersion" + latestVersion); + logger.debug("oldVersion " + getThisJarVersion() + " < latestVersion " + latestVersion); return true; } else if (newVersions[i] < oldVersions[i]) { logger.debug("oldVersion " + getThisJarVersion() + " > latestVersion " + latestVersion); From 726e02347eea3b2fc6949c7756590240a563866f Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 11 Jun 2023 23:32:39 +0200 Subject: [PATCH 358/512] delete unnecessary files --- gradle.properties | 1 - java | 0 release.py | 117 ---------------------------------------------- 3 files changed, 118 deletions(-) delete mode 100644 gradle.properties delete mode 100644 java delete mode 100755 release.py diff --git a/gradle.properties b/gradle.properties deleted file mode 100644 index 7fc6f1ff..00000000 --- a/gradle.properties +++ /dev/null @@ -1 +0,0 @@ -kotlin.code.style=official diff --git a/java b/java deleted file mode 100644 index e69de29b..00000000 diff --git a/release.py b/release.py deleted file mode 100755 index ad099bad..00000000 --- a/release.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python3 - -import re - -import os - -import sys -from hashlib import sha256 -from github import Github -import json -import argparse - -parser = argparse.ArgumentParser(description="Make a new ripme release on github") -parser.add_argument("-f", "--file", help="Path to the version of ripme to release") -parser.add_argument("-t", "--token", help="Your github personal access token") -parser.add_argument("-d", "--debug", help="Run in debug mode", action="store_true") -parser.add_argument("-n", "--non-interactive", help="Do not ask for any input from the user", action="store_true") -parser.add_argument("--test", help="Perform a dry run (Do everything but upload new release)", action="store_true") -parser.add_argument("--skip-hash-check", help="Skip hash check (This should only be used for testing)", action="store_true") -args = parser.parse_args() - -try: - # This binds input to raw_input on python2, we do this because input acts like eval on python2 - input = raw_input -except NameError: - pass - - -# Make sure the file the user selected is a jar -def isJar(filename): - if debug: - print("Checking if {} is a jar file".format(filename)) - return filename.endswith("jar") - - -# Returns true if last entry to the "changeList" section of ripme.json is in the format of $number.$number.$number: and -# false if not -def isValidCommitMessage(message): - if debug: - print(r"Checking if {} matches pattern ^\d+\.\d+\.\d+:".format(message)) - pattern = re.compile(r"^\d+\.\d+\.\d+:") - return re.match(pattern, message) - - -# Checks if the update has the name ripme.jar, if not it renames the file -def checkAndRenameFile(path): - """Check if path (a string) points to a ripme.jar. Returns the possibly renamed file path""" - if not path.endswith("ripme.jar"): - print("Specified file is not named ripme.jar, renaming") - new_path = os.path.join(os.path.dirname(path), "ripme.jar") - os.rename(path, new_path) - return new_path - return path - - -ripmeJson = json.loads(open("ripme.json").read()) -fileToUploadPath = checkAndRenameFile(args.file) -InNoninteractiveMode = args.non_interactive -commitMessage = ripmeJson.get("changeList")[0] -releaseVersion = ripmeJson.get("latestVersion") -debug = args.debug -accessToken = args.token -repoOwner = "ripmeapp" -repoName = "ripme" - -if not os.path.isfile(fileToUploadPath): - print("[!] Error: {} does not exist".format(fileToUploadPath)) - sys.exit(1) - -if not isJar(fileToUploadPath): - print("[!] Error: {} is not a jar file!".format(fileToUploadPath)) - sys.exit(1) - -if not isValidCommitMessage(commitMessage): - print("[!] Error: {} is not a valid commit message as it does not start with a version".format(fileToUploadPath)) - sys.exit(1) - - -if not args.skip_hash_check: - if debug: - print("Reading file {}".format(fileToUploadPath)) - ripmeUpdate = open(fileToUploadPath, mode='rb').read() - - # The actual hash of the file on disk - actualHash = sha256(ripmeUpdate).hexdigest() - - # The hash that we expect the update to have - expectedHash = ripmeJson.get("currentHash") - - # Make sure that the hash of the file we're uploading matches the hash in ripme.json. These hashes not matching will - # cause ripme to refuse to install the update for all users who haven't disabled update hash checking - if expectedHash != actualHash: - print("[!] Error: expected hash of file and actual hash differ") - print("[!] Expected hash is {}".format(expectedHash)) - print("[!] Actual hash is {}".format(actualHash)) - sys.exit(1) -else: - print("[*] WARNING: SKIPPING HASH CHECK") -# Ask the user to review the information before we precede -# This only runs in we're in interactive mode -if not InNoninteractiveMode: - print("File path: {}".format(fileToUploadPath)) - print("Release title: {}".format(commitMessage)) - print("Repo: {}/{}".format(repoOwner, repoName)) - input("\nPlease review the information above and ensure it is correct and then press enter") - -if not args.test: - print("Accessing github using token") - g = Github(accessToken) - - print("Creating release") - release = g.get_user(repoOwner).get_repo(repoName).create_git_release(releaseVersion, commitMessage, "") - - print("Uploading file") - release.upload_asset(fileToUploadPath, "ripme.jar") -else: - print("Not uploading release being script was run with --test flag") From 9a2ee24ab0aeef93a123cfcd905ce51dbae4b251 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 11 Jun 2023 23:54:52 +0200 Subject: [PATCH 359/512] update class for command line properties to DefaultParser, remove unused imports --- src/main/java/com/rarchives/ripme/App.java | 7 ++++--- .../com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 1 - .../rarchives/ripme/ripper/rippers/AerisdiesRipper.java | 1 - .../ripme/ripper/rippers/AllporncomicRipper.java | 1 - .../rarchives/ripme/ripper/rippers/DeviantartRipper.java | 2 -- .../rarchives/ripme/ripper/rippers/DuckmoviesRipper.java | 2 -- .../rarchives/ripme/ripper/rippers/EightmusesRipper.java | 5 ----- .../rarchives/ripme/ripper/rippers/EroShareRipper.java | 1 - .../com/rarchives/ripme/ripper/rippers/EromeRipper.java | 2 -- .../com/rarchives/ripme/ripper/rippers/ErotivRipper.java | 1 - .../ripme/ripper/rippers/FemjoyhunterRipper.java | 1 - .../ripme/ripper/rippers/FitnakedgirlsRipper.java | 1 - .../rarchives/ripme/ripper/rippers/FooktubeRipper.java | 7 ------- .../ripme/ripper/rippers/GfycatporntubeRipper.java | 2 -- .../ripme/ripper/rippers/GirlsOfDesireRipper.java | 1 - .../rarchives/ripme/ripper/rippers/HentaifoxRipper.java | 1 - .../rarchives/ripme/ripper/rippers/MangadexRipper.java | 5 ++--- .../com/rarchives/ripme/ripper/rippers/MrCongRipper.java | 1 - .../java/com/rarchives/ripme/ui/RipStatusComplete.java | 1 - .../ripme/tst/ripper/rippers/XvideosRipperTest.java | 1 - .../java/com/rarchives/ripme/ui/UpdateUtilsTest.java | 9 ++++----- 21 files changed, 10 insertions(+), 43 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index 5456a312..890883ee 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -8,8 +8,9 @@ import com.rarchives.ripme.ui.UpdateUtils; import com.rarchives.ripme.utils.Proxy; import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; -import org.apache.commons.cli.BasicParser; + import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; @@ -285,7 +286,7 @@ public class App { /** * Attempt to rip targetURL. * @param targetURL URL to rip - * @param saveConfig Whether or not you want to save the config (?) + * @param saveConfig Whether you want to save the config (?) */ private static void ripURL(String targetURL, boolean saveConfig) { try { @@ -334,7 +335,7 @@ public class App { * @return CommandLine object containing arguments. */ private static CommandLine getArgs(String[] args) { - BasicParser parser = new BasicParser(); + var parser = new DefaultParser(); try { return parser.parse(getOptions(), args, false); } catch (ParseException e) { diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 1a5d2bb4..83276aea 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -2,7 +2,6 @@ package com.rarchives.ripme.ripper; import java.io.File; import java.io.FileOutputStream; -import java.io.FileWriter; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/AerisdiesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/AerisdiesRipper.java index c68d9784..4558f15e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/AerisdiesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/AerisdiesRipper.java @@ -14,7 +14,6 @@ import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; import java.util.HashMap; public class AerisdiesRipper extends AbstractHTMLRipper { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/AllporncomicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/AllporncomicRipper.java index f0a31ed1..da8c7bd7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/AllporncomicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/AllporncomicRipper.java @@ -12,7 +12,6 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; public class AllporncomicRipper extends AbstractHTMLRipper { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index d701f1f0..3fc34ef3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -18,7 +18,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.regex.Matcher; @@ -28,7 +27,6 @@ import org.jsoup.Connection; import org.jsoup.Connection.Method; import org.jsoup.Connection.Response; import org.jsoup.HttpStatusException; -import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DuckmoviesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DuckmoviesRipper.java index e07cb683..b9cb368b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DuckmoviesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DuckmoviesRipper.java @@ -1,11 +1,9 @@ package com.rarchives.ripme.ripper.rippers; -import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.ripper.AbstractSingleFileRipper; import com.rarchives.ripme.utils.Http; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import java.io.IOException; import java.net.MalformedURLException; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java index f34fb504..c9984569 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java @@ -24,11 +24,6 @@ import com.rarchives.ripme.utils.Http; public class EightmusesRipper extends AbstractHTMLRipper { private Map cookies = new HashMap<>(); - // TODO put up a wiki page on using maps to store titles - // the map for storing the title of each album when downloading sub albums - private Map urlTitles = new HashMap<>(); - - private Boolean rippingSubalbums = false; public EightmusesRipper(URL url) throws IOException { super(url); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java index d4386f3c..bcdca5e4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java @@ -119,7 +119,6 @@ public class EroShareRipper extends AbstractHTMLRipper { for (Element img : imgs) { if (img.hasClass("album-image")) { String imageURL = img.attr("src"); - imageURL = imageURL; URLs.add(imageURL); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java index 8e5e57d5..eb5ad8ea 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java @@ -13,7 +13,6 @@ import com.rarchives.ripme.utils.Utils; import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; @@ -92,7 +91,6 @@ public class EromeRipper extends AbstractHTMLRipper { @Override public List getURLsFromPage(Document doc) { - List URLs = new ArrayList<>(); return getMediaFromPage(doc); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java index 10e73346..a8cd58c2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java @@ -11,7 +11,6 @@ import java.util.regex.Pattern; import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FemjoyhunterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FemjoyhunterRipper.java index 2d1a0eb9..2661d055 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FemjoyhunterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FemjoyhunterRipper.java @@ -12,7 +12,6 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; public class FemjoyhunterRipper extends AbstractHTMLRipper { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FitnakedgirlsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FitnakedgirlsRipper.java index 19afdf3c..3c0a68f0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FitnakedgirlsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FitnakedgirlsRipper.java @@ -13,7 +13,6 @@ import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; public class FitnakedgirlsRipper extends AbstractHTMLRipper { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FooktubeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FooktubeRipper.java index 180dc20c..fed1abe0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FooktubeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FooktubeRipper.java @@ -10,16 +10,9 @@ import java.util.regex.Pattern; import com.rarchives.ripme.ripper.AbstractSingleFileRipper; import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import com.rarchives.ripme.ripper.VideoRipper; -import com.rarchives.ripme.utils.Http; public class FooktubeRipper extends AbstractSingleFileRipper { - private static final String HOST = "mulemax"; - public FooktubeRipper(URL url) throws IOException { super(url); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java index 73669104..bdb58ad2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatporntubeRipper.java @@ -11,8 +11,6 @@ import java.util.regex.Pattern; import com.rarchives.ripme.ripper.AbstractSingleFileRipper; import org.jsoup.nodes.Document; -import com.rarchives.ripme.utils.Http; - public class GfycatporntubeRipper extends AbstractSingleFileRipper { public GfycatporntubeRipper(URL url) throws IOException { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java index 664828f7..a5d14e98 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java @@ -13,7 +13,6 @@ import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; public class GirlsOfDesireRipper extends AbstractHTMLRipper { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java index a6652b41..086596a2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java @@ -12,7 +12,6 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; public class HentaifoxRipper extends AbstractHTMLRipper { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java index d9a80080..1e845d61 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java @@ -126,9 +126,8 @@ public class MangadexRipper extends AbstractJSONRipper { String server; JSONObject chapterJSON = null; TreeMap treeMap = new TreeMap<>(chapterIDs); - Iterator it = treeMap.keySet().iterator(); - while (it.hasNext()) { - double key = (double) it.next(); + for (Double aDouble : treeMap.keySet()) { + double key = (double) aDouble; try { chapterJSON = Http.url(new URL(chapterApiEndPoint + treeMap.get(key))).getJSON(); } catch (IOException e) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MrCongRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MrCongRipper.java index 209db9cf..d0a7b571 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MrCongRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MrCongRipper.java @@ -1,7 +1,6 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.utils.Http; import java.io.IOException; import java.net.MalformedURLException; diff --git a/src/main/java/com/rarchives/ripme/ui/RipStatusComplete.java b/src/main/java/com/rarchives/ripme/ui/RipStatusComplete.java index 418da9d0..2b8058bb 100644 --- a/src/main/java/com/rarchives/ripme/ui/RipStatusComplete.java +++ b/src/main/java/com/rarchives/ripme/ui/RipStatusComplete.java @@ -1,6 +1,5 @@ package com.rarchives.ripme.ui; -import java.io.IOException; import java.nio.file.Path; public class RipStatusComplete { diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XvideosRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XvideosRipperTest.java index 9446b640..3c76c362 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XvideosRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XvideosRipperTest.java @@ -4,7 +4,6 @@ import java.io.IOException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.XvideosRipper; -import com.rarchives.ripme.tst.ripper.rippers.RippersTest; import org.junit.jupiter.api.Test; public class XvideosRipperTest extends RippersTest { diff --git a/src/test/java/com/rarchives/ripme/ui/UpdateUtilsTest.java b/src/test/java/com/rarchives/ripme/ui/UpdateUtilsTest.java index 2f9ba697..d28e6b07 100644 --- a/src/test/java/com/rarchives/ripme/ui/UpdateUtilsTest.java +++ b/src/test/java/com/rarchives/ripme/ui/UpdateUtilsTest.java @@ -7,11 +7,10 @@ public class UpdateUtilsTest { @Test public void testIsNewerVersion() { - UpdateUtils updateUtils = new UpdateUtils(); - Assertions.assertFalse(updateUtils.isNewerVersion("1.7.94")); - Assertions.assertFalse(updateUtils.isNewerVersion("1.7.94-9-asdf")); - Assertions.assertTrue(updateUtils.isNewerVersion("1.7.94-11-asdf")); - Assertions.assertTrue(updateUtils.isNewerVersion("1.7.95")); + Assertions.assertFalse(UpdateUtils.isNewerVersion("1.7.94")); + Assertions.assertFalse(UpdateUtils.isNewerVersion("1.7.94-9-asdf")); + Assertions.assertTrue(UpdateUtils.isNewerVersion("1.7.94-11-asdf")); + Assertions.assertTrue(UpdateUtils.isNewerVersion("1.7.95")); } } \ No newline at end of file From 0b500354cac073399ff7a7f1ff7bc72ce9b65cce Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 12 Jun 2023 00:27:18 +0200 Subject: [PATCH 360/512] new URL(string) replaced with new URI(string).toURL(), as deprecated in java-20 --- .../ripme/ripper/rippers/ListalRipper.java | 470 +++++++++--------- .../java/com/rarchives/ripme/ui/History.java | 4 +- .../ripme/tst/AbstractRipperTest.java | 15 +- 3 files changed, 246 insertions(+), 243 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java index e9f6deef..235da1c7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java @@ -1,234 +1,236 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; -import com.rarchives.ripme.utils.Http; - - - -/** - * @author Tushar - * - */ -public class ListalRipper extends AbstractHTMLRipper { - - private Pattern p1 = Pattern.compile("https:\\/\\/www.listal.com\\/list\\/([a-zA-Z0-9-]+)"); - private Pattern p2 = - Pattern.compile("https:\\/\\/www.listal.com\\/((?:(?:[a-zA-Z0-9-_%]+)\\/?)+)"); - private String listId = null; // listId to get more images via POST. - private String postUrl = "https://www.listal.com/item-list/"; //to load more images. - private UrlType urlType = UrlType.UNKNOWN; - - private DownloadThreadPool listalThreadPool = new DownloadThreadPool("listalThreadPool"); - - public ListalRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getDomain() { - return "listal.com"; - } - - @Override - public String getHost() { - return "listal"; - } - - @Override - public Document getFirstPage() throws IOException { - Document doc = Http.url(url).get(); - if (urlType == UrlType.LIST) { - listId = doc.select("#customlistitems").first().attr("data-listid"); // Used for list types. - } - return doc; - } - - @Override - public List getURLsFromPage(Document page) { - if (urlType == UrlType.LIST) { - // for url of type LIST, https://www.listal.com/list/my-list - return getURLsForListType(page); - } else if (urlType == UrlType.FOLDER) { - // for url of type FOLDER, https://www.listal.com/jim-carrey/pictures - return getURLsForFolderType(page); - } - return null; - } - - @Override - public void downloadURL(URL url, int index) { - listalThreadPool.addThread(new ListalImageDownloadThread(url, index)); - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Matcher m1 = p1.matcher(url.toExternalForm()); - if (m1.matches()) { - // Return the text contained between () in the regex - urlType = UrlType.LIST; - return m1.group(1); - } - - Matcher m2 = p2.matcher(url.toExternalForm()); - if (m2.matches()) { - // Return only gid from capturing group of type listal.com/tvOrSomething/dexter/pictures - urlType = UrlType.FOLDER; - return getFolderTypeGid(m2.group(1)); - } - - throw new MalformedURLException("Expected listal.com URL format: " - + "listal.com/list/my-list-name - got " + url + " instead."); - } - - @Override - public Document getNextPage(Document page) throws IOException { - Document nextPage = super.getNextPage(page); - switch (urlType) { - case LIST: - if (!page.select(".loadmoreitems").isEmpty()) { - // All items are not loaded. - // Load remaining items using postUrl. - - String offSet = page.select(".loadmoreitems").last().attr("data-offset"); - Map postParams = new HashMap<>(); - postParams.put("listid", listId); - postParams.put("offset", offSet); - try { - nextPage = Http.url(postUrl).data(postParams).retries(3).post(); - } catch (IOException e1) { - LOGGER.error("Failed to load more images after " + offSet, e1); - throw e1; - } - } - break; - - case FOLDER: - Elements pageLinks = page.select(".pages a"); - if (!pageLinks.isEmpty() && pageLinks.last().text().startsWith("Next")) { - String nextUrl = pageLinks.last().attr("abs:href"); - nextPage = Http.url(nextUrl).retries(3).get(); - } - break; - - case UNKNOWN: - default: - } - return nextPage; - } - - - @Override - public DownloadThreadPool getThreadPool() { - return listalThreadPool; - } - - /** - * Returns the image urls for UrlType LIST. - */ - private List getURLsForListType(Document page) { - List list = new ArrayList<>(); - for (Element e : page.select(".pure-g a[href*=viewimage]")) { - //list.add("https://www.listal.com" + e.attr("href") + "h"); - list.add(e.attr("abs:href") + "h"); - } - - return list; - } - - /** - * Returns the image urls for UrlType FOLDER. - */ - private List getURLsForFolderType(Document page) { - List list = new ArrayList<>(); - for (Element e : page.select("#browseimagescontainer .imagewrap-outer a")) { - list.add(e.attr("abs:href") + "h"); - } - return list; - } - - /** - * Returns the gid for url type listal.com/tvOrSomething/dexter/pictures - */ - public String getFolderTypeGid(String group) throws MalformedURLException { - String[] folders = group.split("/"); - try { - if (folders.length == 2 && folders[1].equals("pictures")) { - // Url is probably for an actor. - return folders[0]; - } - - if (folders.length == 3 && folders[2].equals("pictures")) { - // Url if for a folder(like movies, tv etc). - Document doc = Http.url(url).get(); - return doc.select(".itemheadingmedium").first().text(); - } - - } catch (Exception e) { - LOGGER.error(e); - } - throw new MalformedURLException("Unable to fetch the gid for given url."); - } - - private class ListalImageDownloadThread implements Runnable { - - private final URL url; - private final int index; - - public ListalImageDownloadThread(URL url, int index) { - super(); - this.url = url; - this.index = index; - } - - @Override - public void run() { - getImage(); - } - - public void getImage() { - try { - Document doc = Http.url(url).get(); - - String imageUrl = doc.getElementsByClass("pure-img").attr("src"); - if (imageUrl != "") { - addURLToDownload(new URL(imageUrl), getPrefix(index), "", null, null, - getImageName()); - } else { - LOGGER.error("Couldnt find image from url: " + url); - } - } catch (IOException e) { - LOGGER.error("[!] Exception while downloading image: " + url, e); - } - } - - public String getImageName() { - // Returns the image number of the link if possible. - String name = this.url.toExternalForm(); - try { - name = name.substring(name.lastIndexOf("/") + 1); - } catch (Exception e) { - LOGGER.info("Failed to get name for the image."); - name = null; - } - // Listal stores images as .jpg - return name + ".jpg"; - } - } - - private static enum UrlType { - LIST, FOLDER, UNKNOWN - } -} +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.ripper.DownloadThreadPool; +import com.rarchives.ripme.utils.Http; + + + +/** + * @author Tushar + * + */ +public class ListalRipper extends AbstractHTMLRipper { + + private Pattern p1 = Pattern.compile("https:\\/\\/www.listal.com\\/list\\/([a-zA-Z0-9-]+)"); + private Pattern p2 = + Pattern.compile("https:\\/\\/www.listal.com\\/((?:(?:[a-zA-Z0-9-_%]+)\\/?)+)"); + private String listId = null; // listId to get more images via POST. + private String postUrl = "https://www.listal.com/item-list/"; //to load more images. + private UrlType urlType = UrlType.UNKNOWN; + + private DownloadThreadPool listalThreadPool = new DownloadThreadPool("listalThreadPool"); + + public ListalRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getDomain() { + return "listal.com"; + } + + @Override + public String getHost() { + return "listal"; + } + + @Override + public Document getFirstPage() throws IOException { + Document doc = Http.url(url).get(); + if (urlType == UrlType.LIST) { + listId = doc.select("#customlistitems").first().attr("data-listid"); // Used for list types. + } + return doc; + } + + @Override + public List getURLsFromPage(Document page) { + if (urlType == UrlType.LIST) { + // for url of type LIST, https://www.listal.com/list/my-list + return getURLsForListType(page); + } else if (urlType == UrlType.FOLDER) { + // for url of type FOLDER, https://www.listal.com/jim-carrey/pictures + return getURLsForFolderType(page); + } + return null; + } + + @Override + public void downloadURL(URL url, int index) { + listalThreadPool.addThread(new ListalImageDownloadThread(url, index)); + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Matcher m1 = p1.matcher(url.toExternalForm()); + if (m1.matches()) { + // Return the text contained between () in the regex + urlType = UrlType.LIST; + return m1.group(1); + } + + Matcher m2 = p2.matcher(url.toExternalForm()); + if (m2.matches()) { + // Return only gid from capturing group of type listal.com/tvOrSomething/dexter/pictures + urlType = UrlType.FOLDER; + return getFolderTypeGid(m2.group(1)); + } + + throw new MalformedURLException("Expected listal.com URL format: " + + "listal.com/list/my-list-name - got " + url + " instead."); + } + + @Override + public Document getNextPage(Document page) throws IOException { + Document nextPage = super.getNextPage(page); + switch (urlType) { + case LIST: + if (!page.select(".loadmoreitems").isEmpty()) { + // All items are not loaded. + // Load remaining items using postUrl. + + String offSet = page.select(".loadmoreitems").last().attr("data-offset"); + Map postParams = new HashMap<>(); + postParams.put("listid", listId); + postParams.put("offset", offSet); + try { + nextPage = Http.url(postUrl).data(postParams).retries(3).post(); + } catch (IOException e1) { + LOGGER.error("Failed to load more images after " + offSet, e1); + throw e1; + } + } + break; + + case FOLDER: + Elements pageLinks = page.select(".pages a"); + if (!pageLinks.isEmpty() && pageLinks.last().text().startsWith("Next")) { + String nextUrl = pageLinks.last().attr("abs:href"); + nextPage = Http.url(nextUrl).retries(3).get(); + } + break; + + case UNKNOWN: + default: + } + return nextPage; + } + + + @Override + public DownloadThreadPool getThreadPool() { + return listalThreadPool; + } + + /** + * Returns the image urls for UrlType LIST. + */ + private List getURLsForListType(Document page) { + List list = new ArrayList<>(); + for (Element e : page.select(".pure-g a[href*=viewimage]")) { + //list.add("https://www.listal.com" + e.attr("href") + "h"); + list.add(e.attr("abs:href") + "h"); + } + + return list; + } + + /** + * Returns the image urls for UrlType FOLDER. + */ + private List getURLsForFolderType(Document page) { + List list = new ArrayList<>(); + for (Element e : page.select("#browseimagescontainer .imagewrap-outer a")) { + list.add(e.attr("abs:href") + "h"); + } + return list; + } + + /** + * Returns the gid for url type listal.com/tvOrSomething/dexter/pictures + */ + public String getFolderTypeGid(String group) throws MalformedURLException { + String[] folders = group.split("/"); + try { + if (folders.length == 2 && folders[1].equals("pictures")) { + // Url is probably for an actor. + return folders[0]; + } + + if (folders.length == 3 && folders[2].equals("pictures")) { + // Url if for a folder(like movies, tv etc). + Document doc = Http.url(url).get(); + return doc.select(".itemheadingmedium").first().text(); + } + + } catch (Exception e) { + LOGGER.error(e); + } + throw new MalformedURLException("Unable to fetch the gid for given url."); + } + + private class ListalImageDownloadThread implements Runnable { + + private final URL url; + private final int index; + + public ListalImageDownloadThread(URL url, int index) { + super(); + this.url = url; + this.index = index; + } + + @Override + public void run() { + getImage(); + } + + public void getImage() { + try { + Document doc = Http.url(url).get(); + + String imageUrl = doc.getElementsByClass("pure-img").attr("src"); + if (imageUrl != "") { + addURLToDownload(new URI(imageUrl).toURL(), getPrefix(index), "", null, null, + getImageName()); + } else { + LOGGER.error("Couldnt find image from url: " + url); + } + } catch (IOException | URISyntaxException e) { + LOGGER.error("[!] Exception while downloading image: " + url, e); + } + } + + public String getImageName() { + // Returns the image number of the link if possible. + String name = this.url.toExternalForm(); + try { + name = name.substring(name.lastIndexOf("/") + 1); + } catch (Exception e) { + LOGGER.info("Failed to get name for the image."); + name = null; + } + // Listal stores images as .jpg + return name + ".jpg"; + } + } + + private static enum UrlType { + LIST, FOLDER, UNKNOWN + } +} diff --git a/src/main/java/com/rarchives/ripme/ui/History.java b/src/main/java/com/rarchives/ripme/ui/History.java index f3f9451f..190eeeb8 100644 --- a/src/main/java/com/rarchives/ripme/ui/History.java +++ b/src/main/java/com/rarchives/ripme/ui/History.java @@ -100,7 +100,7 @@ public class History { public void fromFile(String filename) throws IOException { try (InputStream is = new FileInputStream(filename)) { - String jsonString = IOUtils.toString(is); + String jsonString = IOUtils.toString(is, "UTF-8"); JSONArray jsonArray = new JSONArray(jsonString); fromJSON(jsonArray); } catch (JSONException e) { @@ -134,7 +134,7 @@ public class History { public void toFile(String filename) throws IOException { try (OutputStream os = new FileOutputStream(filename)) { - IOUtils.write(toJSON().toString(2), os); + IOUtils.write(toJSON().toString(2), os, "UTF-8"); } } } diff --git a/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java b/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java index c750b22b..7eb3df43 100644 --- a/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/AbstractRipperTest.java @@ -4,7 +4,8 @@ import com.rarchives.ripme.ripper.AbstractRipper; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -12,20 +13,20 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class AbstractRipperTest { @Test - public void testGetFileName() throws IOException { - String fileName = AbstractRipper.getFileName(new URL("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D"),null, "test", "test"); + public void testGetFileName() throws IOException, URISyntaxException { + String fileName = AbstractRipper.getFileName(new URI("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D").toURL(),null, "test", "test"); assertEquals("test.test", fileName); - fileName = AbstractRipper.getFileName(new URL("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D"), null,"test", null); + fileName = AbstractRipper.getFileName(new URI("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D").toURL(), null,"test", null); assertEquals("test", fileName); - fileName = AbstractRipper.getFileName(new URL("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D"), null,null, null); + fileName = AbstractRipper.getFileName(new URI("http://www.tsumino.com/Image/Object?name=U1EieteEGwm6N1dGszqCpA%3D%3D").toURL(), null,null, null); assertEquals("Object", fileName); - fileName = AbstractRipper.getFileName(new URL("http://www.test.com/file.png"), null,null, null); + fileName = AbstractRipper.getFileName(new URI("http://www.test.com/file.png").toURL(), null,null, null); assertEquals("file.png", fileName); - fileName = AbstractRipper.getFileName(new URL("http://www.test.com/file."), null,null, null); + fileName = AbstractRipper.getFileName(new URI("http://www.test.com/file.").toURL(), null,null, null); assertEquals("file.", fileName); } From 8cbdb33f04e8c3905dd90601c10d83916ee1c5da Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 12 Jun 2023 19:42:52 +0200 Subject: [PATCH 361/512] crfl --> cr, arttnruppertest --- .../tst/ripper/rippers/ArtstnRipperTest.java | 54 ++++++++++--------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtstnRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtstnRipperTest.java index 8817b369..ee8621c2 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtstnRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtstnRipperTest.java @@ -1,26 +1,28 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URL; - -import com.rarchives.ripme.ripper.rippers.ArtstnRipper; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -public class ArtstnRipperTest extends RippersTest { - @Test - @Tag("flaky") - public void testSingleProject() throws IOException { - URL url = new URL("https://artstn.co/p/JlE15Z"); - testRipper(new ArtstnRipper(url)); - } - - @Test - @Disabled("Failed with cloudflare protection") - public void testUserPortfolio() throws IOException { - URL url = new URL("https://artstn.co/m/rv37"); - testRipper(new ArtstnRipper(url)); - } -} +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.ArtstnRipper; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +public class ArtstnRipperTest extends RippersTest { + @Test + @Tag("flaky") + public void testSingleProject() throws IOException, URISyntaxException { + URL url = new URI("https://artstn.co/p/JlE15Z").toURL(); + testRipper(new ArtstnRipper(url)); + } + + @Test + @Disabled("Failed with cloudflare protection") + public void testUserPortfolio() throws IOException, URISyntaxException { + URL url = new URI("https://artstn.co/m/rv37").toURL(); + testRipper(new ArtstnRipper(url)); + } +} From 53e8a66e4c8cee1395bd6b84254bd8ba67ee96f6 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 12 Jun 2023 01:31:50 +0200 Subject: [PATCH 362/512] new URL(string) replaced with new URI(string).toURL(), as deprecated in java-20 --- .../ripper/rippers/AerisdiesRipperTest.java | 18 ++++++----- .../rippers/AllporncomicRipperTest.java | 7 +++-- .../ripper/rippers/ArtAlleyRipperTest.java | 7 +++-- .../ripper/rippers/ArtStationRipperTest.java | 18 ++++++----- .../tst/ripper/rippers/BaraagRipperTest.java | 7 +++-- .../tst/ripper/rippers/BatoRipperTest.java | 14 +++++---- .../tst/ripper/rippers/BcfakesRipperTest.java | 7 +++-- .../rippers/BlackbrickroadofozRipperTest.java | 7 +++-- .../tst/ripper/rippers/BooruRipperTest.java | 26 +++++++++------- .../tst/ripper/rippers/CfakeRipperTest.java | 7 +++-- .../tst/ripper/rippers/ChanRipperTest.java | 30 +++++++++--------- .../ripper/rippers/CheveretoRipperTest.java | 11 ++++--- .../ripper/rippers/ComicextraRipperTest.java | 10 +++--- .../ripper/rippers/CyberdropRipperTest.java | 14 +++++---- .../ripper/rippers/DanbooruRipperTest.java | 18 ++++++----- .../tst/ripper/rippers/E621RipperTest.java | 31 ++++++++++--------- 16 files changed, 126 insertions(+), 106 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java index a8fec2e7..4c0bd833 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.AerisdiesRipper; @@ -12,29 +14,29 @@ import org.junit.jupiter.api.Test; public class AerisdiesRipperTest extends RippersTest { @Test @Tag("flaky") - public void testAlbum() throws IOException { - AerisdiesRipper ripper = new AerisdiesRipper(new URL("http://www.aerisdies.com/html/lb/alb_1097_1.html")); + public void testAlbum() throws IOException, URISyntaxException { + AerisdiesRipper ripper = new AerisdiesRipper(new URI("http://www.aerisdies.com/html/lb/alb_1097_1.html").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testSubAlbum() throws IOException { - AerisdiesRipper ripper = new AerisdiesRipper(new URL("http://www.aerisdies.com/html/lb/alb_3692_1.html")); + public void testSubAlbum() throws IOException, URISyntaxException { + AerisdiesRipper ripper = new AerisdiesRipper(new URI("http://www.aerisdies.com/html/lb/alb_3692_1.html").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testDjAlbum() throws IOException { - AerisdiesRipper ripper = new AerisdiesRipper(new URL("http://www.aerisdies.com/html/lb/douj_5230_1.html")); + public void testDjAlbum() throws IOException, URISyntaxException { + AerisdiesRipper ripper = new AerisdiesRipper(new URI("http://www.aerisdies.com/html/lb/douj_5230_1.html").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testGetGID() throws IOException { - URL url = new URL("http://www.aerisdies.com/html/lb/douj_5230_1.html"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("http://www.aerisdies.com/html/lb/douj_5230_1.html").toURL(); AerisdiesRipper ripper = new AerisdiesRipper(url); Assertions.assertEquals("5230", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AllporncomicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AllporncomicRipperTest.java index 41c9542a..20b79c0b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AllporncomicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/AllporncomicRipperTest.java @@ -5,13 +5,14 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; public class AllporncomicRipperTest extends RippersTest { @Test @Tag("flaky") - public void testAlbum() throws IOException { - AllporncomicRipper ripper = new AllporncomicRipper(new URL("https://allporncomic.com/porncomic/dnd-pvp-dungeons-dragons-fred-perry/1-dnd-pvp")); + public void testAlbum() throws IOException, URISyntaxException { + AllporncomicRipper ripper = new AllporncomicRipper(new URI("https://allporncomic.com/porncomic/dnd-pvp-dungeons-dragons-fred-perry/1-dnd-pvp").toURL()); testRipper(ripper); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtAlleyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtAlleyRipperTest.java index 400e95c3..63b9d69b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtAlleyRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtAlleyRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.ArtAlleyRipper; import org.junit.jupiter.api.Disabled; @@ -10,8 +11,8 @@ import org.junit.jupiter.api.Test; public class ArtAlleyRipperTest extends RippersTest { @Test @Disabled("website switched off") - public void testRip() throws IOException { - ArtAlleyRipper ripper = new ArtAlleyRipper(new URL("https://artalley.social/@curator/media")); + public void testRip() throws IOException, URISyntaxException { + ArtAlleyRipper ripper = new ArtAlleyRipper(new URI("https://artalley.social/@curator/media").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java index e29a32ed..6450cad1 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -14,11 +16,11 @@ public class ArtStationRipperTest extends RippersTest { @Test @Tag("flaky") - public void testArtStationProjects() throws IOException { + public void testArtStationProjects() throws IOException, URISyntaxException { List contentURLs = new ArrayList<>(); - contentURLs.add(new URL("https://www.artstation.com/artwork/the-dwarf-mortar")); - contentURLs.add(new URL("https://www.artstation.com/artwork/K36GR")); - contentURLs.add(new URL("http://artstation.com/artwork/5JJQw")); + contentURLs.add(new URI("https://www.artstation.com/artwork/the-dwarf-mortar").toURL()); + contentURLs.add(new URI("https://www.artstation.com/artwork/K36GR").toURL()); + contentURLs.add(new URI("http://artstation.com/artwork/5JJQw").toURL()); for (URL url : contentURLs) { ArtStationRipper ripper = new ArtStationRipper(url); testRipper(ripper); @@ -27,11 +29,11 @@ public class ArtStationRipperTest extends RippersTest { @Test @Tag("flaky") - public void testArtStationUserProfiles() throws IOException { + public void testArtStationUserProfiles() throws IOException, URISyntaxException { List contentURLs = new ArrayList<>(); - contentURLs.add(new URL("https://www.artstation.com/heitoramatsu")); - contentURLs.add(new URL("https://artstation.com/kuvshinov_ilya")); - contentURLs.add(new URL("http://artstation.com/givemeapiggy")); + contentURLs.add(new URI("https://www.artstation.com/heitoramatsu").toURL()); + contentURLs.add(new URI("https://artstation.com/kuvshinov_ilya").toURL()); + contentURLs.add(new URI("http://artstation.com/givemeapiggy").toURL()); for (URL url : contentURLs) { ArtStationRipper ripper = new ArtStationRipper(url); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BaraagRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BaraagRipperTest.java index 5e71de70..7b987b74 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BaraagRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BaraagRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.BaraagRipper; import org.junit.jupiter.api.Tag; @@ -10,8 +11,8 @@ import org.junit.jupiter.api.Test; public class BaraagRipperTest extends RippersTest { @Test @Tag("flaky") - public void testRip() throws IOException { - BaraagRipper ripper = new BaraagRipper(new URL("https://baraag.net/@darkshadow777/media")); + public void testRip() throws IOException, URISyntaxException { + BaraagRipper ripper = new BaraagRipper(new URI("https://baraag.net/@darkshadow777/media").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java index 3ceb2ac7..6849f0e1 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BatoRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.BatoRipper; @@ -13,22 +15,22 @@ public class BatoRipperTest extends RippersTest { @Test @Disabled("cloudlare? gets unavailable in test but works in browser") - public void testRip() throws IOException { - BatoRipper ripper = new BatoRipper(new URL("https://bato.to/chapter/1207152")); + public void testRip() throws IOException, URISyntaxException { + BatoRipper ripper = new BatoRipper(new URI("https://bato.to/chapter/1207152").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("https://bato.to/chapter/1207152"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://bato.to/chapter/1207152").toURL(); BatoRipper ripper = new BatoRipper(url); Assertions.assertEquals("1207152", ripper.getGID(url)); } @Test @Disabled("cloudlare? gets unavailable in test but works in browser") - public void testGetAlbumTitle() throws IOException { - URL url = new URL("https://bato.to/chapter/1207152"); + public void testGetAlbumTitle() throws IOException, URISyntaxException { + URL url = new URI("https://bato.to/chapter/1207152").toURL(); BatoRipper ripper = new BatoRipper(url); Assertions.assertEquals("bato_1207152_I_Messed_Up_by_Teaching_at_a_Black_Gyaru_School!_Ch.2", ripper.getAlbumTitle(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BcfakesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BcfakesRipperTest.java index 8c31ffd4..3140c056 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BcfakesRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BcfakesRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.BcfakesRipper; @@ -11,8 +12,8 @@ import org.junit.jupiter.api.Test; public class BcfakesRipperTest extends RippersTest { @Test @Disabled("21/06/2018 This test was disbaled as the site has experienced notable downtime") - public void testRip() throws IOException { - BcfakesRipper ripper = new BcfakesRipper(new URL("http://www.bcfakes.com/celebritylist/olivia-wilde/")); + public void testRip() throws IOException, URISyntaxException { + BcfakesRipper ripper = new BcfakesRipper(new URI("http://www.bcfakes.com/celebritylist/olivia-wilde/").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BlackbrickroadofozRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BlackbrickroadofozRipperTest.java index 0bf11d58..c28cc52d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BlackbrickroadofozRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BlackbrickroadofozRipperTest.java @@ -6,14 +6,15 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; public class BlackbrickroadofozRipperTest extends RippersTest { @Test @Disabled("Commented out on 02/04/2019 because the serve has been down for a while") - public void testRip() throws IOException { + public void testRip() throws IOException, URISyntaxException { BlackbrickroadofozRipper ripper = new BlackbrickroadofozRipper( - new URL("http://www.blackbrickroadofoz.com/comic/beginning")); + new URI("http://www.blackbrickroadofoz.com/comic/beginning").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BooruRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BooruRipperTest.java index 663418b4..89efef4e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BooruRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/BooruRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -13,10 +15,10 @@ import org.junit.jupiter.api.Test; public class BooruRipperTest extends RippersTest { @Test @Tag("flaky") - public void testRip() throws IOException { + public void testRip() throws IOException, URISyntaxException { List passURLs = new ArrayList<>(); - passURLs.add(new URL("https://xbooru.com/index.php?page=post&s=list&tags=furry")); - passURLs.add(new URL("https://gelbooru.com/index.php?page=post&s=list&tags=animal_ears")); + passURLs.add(new URI("https://xbooru.com/index.php?page=post&s=list&tags=furry").toURL()); + passURLs.add(new URI("https://gelbooru.com/index.php?page=post&s=list&tags=animal_ears").toURL()); for (URL url : passURLs) { BooruRipper ripper = new BooruRipper(url); @@ -25,9 +27,9 @@ public class BooruRipperTest extends RippersTest { } @Test - public void testGetGID() throws IOException { - URL xbooruUrl = new URL("https://xbooru.com/index.php?page=post&s=list&tags=furry"); - URL gelbooruUrl = new URL("https://gelbooru.com/index.php?page=post&s=list&tags=animal_ears"); + public void testGetGID() throws IOException, URISyntaxException { + URL xbooruUrl = new URI("https://xbooru.com/index.php?page=post&s=list&tags=furry").toURL(); + URL gelbooruUrl = new URI("https://gelbooru.com/index.php?page=post&s=list&tags=animal_ears").toURL(); BooruRipper xbooruRipper = new BooruRipper(xbooruUrl); BooruRipper gelbooruRipper = new BooruRipper(gelbooruUrl); @@ -37,9 +39,9 @@ public class BooruRipperTest extends RippersTest { } @Test - public void testGetDomain() throws IOException { - URL xbooruUrl = new URL("https://xbooru.com/index.php?page=post&s=list&tags=furry"); - URL gelbooruUrl = new URL("https://gelbooru.com/index.php?page=post&s=list&tags=animal_ears"); + public void testGetDomain() throws IOException, URISyntaxException { + URL xbooruUrl = new URI("https://xbooru.com/index.php?page=post&s=list&tags=furry").toURL(); + URL gelbooruUrl = new URI("https://gelbooru.com/index.php?page=post&s=list&tags=animal_ears").toURL(); BooruRipper xbooruRipper = new BooruRipper(xbooruUrl); BooruRipper gelbooruRipper = new BooruRipper(gelbooruUrl); @@ -49,9 +51,9 @@ public class BooruRipperTest extends RippersTest { } @Test - public void testGetHost() throws IOException { - URL xbooruUrl = new URL("https://xbooru.com/index.php?page=post&s=list&tags=furry"); - URL gelbooruUrl = new URL("https://gelbooru.com/index.php?page=post&s=list&tags=animal_ears"); + public void testGetHost() throws IOException, URISyntaxException { + URL xbooruUrl = new URI("https://xbooru.com/index.php?page=post&s=list&tags=furry").toURL(); + URL gelbooruUrl = new URI("https://gelbooru.com/index.php?page=post&s=list&tags=animal_ears").toURL(); BooruRipper xbooruRipper = new BooruRipper(xbooruUrl); BooruRipper gelbooruRipper = new BooruRipper(gelbooruUrl); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CfakeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CfakeRipperTest.java index 3e6dad94..95f7ec2e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CfakeRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CfakeRipperTest.java @@ -1,13 +1,14 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.CfakeRipper; public class CfakeRipperTest extends RippersTest { - public void testRip() throws IOException { - CfakeRipper ripper = new CfakeRipper(new URL("http://cfake.com/picture/Zooey_Deschanel/1264")); + public void testRip() throws IOException, URISyntaxException { + CfakeRipper ripper = new CfakeRipper(new URI("http://cfake.com/picture/Zooey_Deschanel/1264").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java index e7b285fc..078e366d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; @@ -15,14 +17,14 @@ import org.junit.jupiter.api.Test; public class ChanRipperTest extends RippersTest { @Test - public void testChanURLPasses() throws IOException { + public void testChanURLPasses() throws IOException, URISyntaxException { List passURLs = new ArrayList<>(); // URLs that should work - passURLs.add(new URL("http://desuchan.net/v/res/7034.html")); - passURLs.add(new URL("https://boards.4chan.org/hr/thread/3015701")); - passURLs.add(new URL("https://boards.420chan.org/420/res/232066.php")); - passURLs.add(new URL("http://7chan.org/gif/res/25873.html")); - passURLs.add(new URL("https://rbt.asia/g/thread/70643087/")); //must work with TLDs with len of 4 + passURLs.add(new URI("http://desuchan.net/v/res/7034.html").toURL()); + passURLs.add(new URI("https://boards.4chan.org/hr/thread/3015701").toURL()); + passURLs.add(new URI("https://boards.420chan.org/420/res/232066.php").toURL()); + passURLs.add(new URI("http://7chan.org/gif/res/25873.html").toURL()); + passURLs.add(new URI("https://rbt.asia/g/thread/70643087/").toURL()); //must work with TLDs with len of 4 for (URL url : passURLs) { ChanRipper ripper = new ChanRipper(url); ripper.setup(); @@ -32,13 +34,13 @@ public class ChanRipperTest extends RippersTest { } } @Test - public void testChanStringParsing() throws IOException { + public void testChanStringParsing() throws IOException, URISyntaxException { List site1 = Arrays.asList("site1.com"); List site1Cdns = Arrays.asList("cnd1.site1.com", "cdn2.site2.biz"); List site2 = Arrays.asList("site2.co.uk"); List site2Cdns = Arrays.asList("cdn.site2.co.uk"); - ChanRipper ripper = new ChanRipper(new URL("http://desuchan.net/v/res/7034.html")); + ChanRipper ripper = new ChanRipper(new URI("http://desuchan.net/v/res/7034.html").toURL()); List chansFromConfig = ripper .getChansFromConfig("site1.com[cnd1.site1.com|cdn2.site2.biz],site2.co.uk[cdn.site2.co.uk]"); Assertions.assertEquals(chansFromConfig.get(0).getDomains(), site1); @@ -48,22 +50,18 @@ public class ChanRipperTest extends RippersTest { Assertions.assertEquals(chansFromConfig.get(1).getCdns(), site2Cdns); } @Test - public void testChanRipper() throws IOException { + public void testChanRipper() throws IOException, URISyntaxException { List contentURLs = new ArrayList<>(); - contentURLs.add(new URL(getRandomThreadDesuarchive())); + contentURLs.add(new URI(getRandomThreadDesuarchive()).toURL()); for (URL url : contentURLs) { ChanRipper ripper = new ChanRipper(url); testChanRipper(ripper); } } - /** - * - * @return String returns a url to a active desuarchive.org tread as a string - */ - public String getRandomThreadDesuarchive() { + public String getRandomThreadDesuarchive() throws URISyntaxException { try { - Document doc = Http.url(new URL("https://desuarchive.org/wsg/")).get(); + Document doc = Http.url(new URI("https://desuarchive.org/wsg/").toURL()).get(); return doc.select("div.post_data > a").first().attr("href"); } catch (IOException e) { e.printStackTrace(); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java index 5b233431..6d893527 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.CheveretoRipper; import org.junit.jupiter.api.Tag; @@ -10,14 +11,14 @@ import org.junit.jupiter.api.Test; public class CheveretoRipperTest extends RippersTest { @Test @Tag("flaky") - public void testTagFox() throws IOException { - CheveretoRipper ripper = new CheveretoRipper(new URL("http://tag-fox.com/album/Thjb")); + public void testTagFox() throws IOException, URISyntaxException { + CheveretoRipper ripper = new CheveretoRipper(new URI("http://tag-fox.com/album/Thjb").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testSubdirAlbum() throws IOException { - CheveretoRipper ripper = new CheveretoRipper(new URL("https://kenzato.uk/booru/album/TnEc")); + public void testSubdirAlbum() throws IOException, URISyntaxException { + CheveretoRipper ripper = new CheveretoRipper(new URI("https://kenzato.uk/booru/album/TnEc").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java index 75c1f955..0a64487d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ComicextraRipper; import org.junit.jupiter.api.Disabled; @@ -10,15 +12,15 @@ import org.junit.jupiter.api.Test; public class ComicextraRipperTest extends RippersTest { @Test @Tag("flaky") - public void testComicUrl() throws IOException { - URL url = new URL("https://www.comicextra.com/comic/karma-police"); + public void testComicUrl() throws IOException, URISyntaxException { + URL url = new URI("https://www.comicextra.com/comic/karma-police").toURL(); ComicextraRipper ripper = new ComicextraRipper(url); testRipper(ripper); } @Test @Disabled("no images found error, broken ripper?") - public void testChapterUrl() throws IOException { - URL url = new URL("https://www.comicextra.com/v-for-vendetta/chapter-1"); + public void testChapterUrl() throws IOException, URISyntaxException { + URL url = new URI("https://www.comicextra.com/v-for-vendetta/chapter-1").toURL(); ComicextraRipper ripper = new ComicextraRipper(url); testRipper(ripper); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java index 847f2abf..17f0c8a6 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java @@ -8,6 +8,8 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; @@ -16,11 +18,11 @@ import java.util.Map; public class CyberdropRipperTest extends RippersTest { @Test - public void testScrolllerGID() throws IOException { + public void testScrolllerGID() throws IOException, URISyntaxException { Map testURLs = new HashMap<>(); - testURLs.put(new URL("https://cyberdrop.me/a/n4umdBjw"), "n4umdBjw"); - testURLs.put(new URL("https://cyberdrop.me/a/iLtp4BjW"), "iLtp4BjW"); + testURLs.put(new URI("https://cyberdrop.me/a/n4umdBjw").toURL(), "n4umdBjw"); + testURLs.put(new URI("https://cyberdrop.me/a/iLtp4BjW").toURL(), "iLtp4BjW"); for (URL url : testURLs.keySet()) { CyberdropRipper ripper = new CyberdropRipper(url); ripper.setup(); @@ -31,11 +33,11 @@ public class CyberdropRipperTest extends RippersTest { @Test @Tag("flaky") - public void testCyberdropNumberOfFiles() throws IOException { + public void testCyberdropNumberOfFiles() throws IOException, URISyntaxException { List testURLs = new ArrayList(); - testURLs.add(new URL("https://cyberdrop.me/a/n4umdBjw")); - testURLs.add(new URL("https://cyberdrop.me/a/iLtp4BjW")); + testURLs.add(new URI("https://cyberdrop.me/a/n4umdBjw").toURL()); + testURLs.add(new URI("https://cyberdrop.me/a/iLtp4BjW").toURL()); for (URL url : testURLs) { Assertions.assertTrue(willDownloadAllFiles(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DanbooruRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DanbooruRipperTest.java index de3c6b5e..dd6e7163 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DanbooruRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DanbooruRipperTest.java @@ -6,6 +6,8 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -13,10 +15,10 @@ import java.util.List; public class DanbooruRipperTest extends RippersTest { @Test @Tag("flaky") - public void testRip() throws IOException { + public void testRip() throws IOException, URISyntaxException { List passURLs = new ArrayList<>(); - passURLs.add(new URL("https://danbooru.donmai.us/posts?tags=brown_necktie")); - passURLs.add(new URL("https://danbooru.donmai.us/posts?page=1&tags=pink_sweater_vest")); + passURLs.add(new URI("https://danbooru.donmai.us/posts?tags=brown_necktie").toURL()); + passURLs.add(new URI("https://danbooru.donmai.us/posts?page=1&tags=pink_sweater_vest").toURL()); for (URL url : passURLs) { DanbooruRipper danbooruRipper = new DanbooruRipper(url); @@ -25,9 +27,9 @@ public class DanbooruRipperTest extends RippersTest { } @Test - public void testGetGID() throws IOException { - URL danBooruUrl = new URL("https://danbooru.donmai.us/posts?tags=brown_necktie"); - URL danBooruUrl2 = new URL("https://danbooru.donmai.us/posts?page=1&tags=pink_sweater_vest"); + public void testGetGID() throws IOException, URISyntaxException { + URL danBooruUrl = new URI("https://danbooru.donmai.us/posts?tags=brown_necktie").toURL(); + URL danBooruUrl2 = new URI("https://danbooru.donmai.us/posts?page=1&tags=pink_sweater_vest").toURL(); DanbooruRipper danbooruRipper = new DanbooruRipper(danBooruUrl); DanbooruRipper danbooruRipper2 = new DanbooruRipper(danBooruUrl2); @@ -37,8 +39,8 @@ public class DanbooruRipperTest extends RippersTest { } @Test - public void testGetHost() throws IOException { - URL danBooruUrl = new URL("https://danbooru.donmai.us/posts?tags=brown_necktie"); + public void testGetHost() throws IOException, URISyntaxException { + URL danBooruUrl = new URI("https://danbooru.donmai.us/posts?tags=brown_necktie").toURL(); DanbooruRipper danbooruRipper = new DanbooruRipper(danBooruUrl); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java index 31ce2260..4859ade0 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.E621Ripper; import org.junit.jupiter.api.Assertions; @@ -9,20 +10,20 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class E621RipperTest extends RippersTest { - public void testRip() throws IOException { - E621Ripper ripper = new E621Ripper(new URL("https://e621.net/posts?tags=beach")); + public void testRip() throws IOException, URISyntaxException { + E621Ripper ripper = new E621Ripper(new URI("https://e621.net/posts?tags=beach").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testFlashOrWebm() throws IOException { - E621Ripper ripper = new E621Ripper(new URL("https://e621.net/posts?page=4&tags=gif+rating%3As+3d")); + public void testFlashOrWebm() throws IOException, URISyntaxException { + E621Ripper ripper = new E621Ripper(new URI("https://e621.net/posts?page=4&tags=gif+rating%3As+3d").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testGetNextPage() throws IOException { - E621Ripper nextPageRipper = new E621Ripper(new URL("https://e621.net/posts?tags=cosmicminerals")); + public void testGetNextPage() throws IOException, URISyntaxException { + E621Ripper nextPageRipper = new E621Ripper(new URI("https://e621.net/posts?tags=cosmicminerals").toURL()); try { nextPageRipper.getNextPage(nextPageRipper.getFirstPage()); assert (true); @@ -30,7 +31,7 @@ public class E621RipperTest extends RippersTest { throw e; } - E621Ripper noNextPageRipper = new E621Ripper(new URL("https://e621.net/post/index/1/cosmicminerals")); + E621Ripper noNextPageRipper = new E621Ripper(new URI("https://e621.net/post/index/1/cosmicminerals").toURL()); try { noNextPageRipper.getNextPage(noNextPageRipper.getFirstPage()); } catch (IOException e) { @@ -39,20 +40,20 @@ public class E621RipperTest extends RippersTest { } @Test @Tag("flaky") - public void testOldRip() throws IOException { - E621Ripper ripper = new E621Ripper(new URL("https://e621.net/post/index/1/beach")); + public void testOldRip() throws IOException, URISyntaxException { + E621Ripper ripper = new E621Ripper(new URI("https://e621.net/post/index/1/beach").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testOldFlashOrWebm() throws IOException { - E621Ripper ripper = new E621Ripper(new URL("https://e621.net/post/index/1/gif")); + public void testOldFlashOrWebm() throws IOException, URISyntaxException { + E621Ripper ripper = new E621Ripper(new URI("https://e621.net/post/index/1/gif").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testOldGetNextPage() throws IOException { - E621Ripper nextPageRipper = new E621Ripper(new URL("https://e621.net/post/index/1/cosmicminerals")); + public void testOldGetNextPage() throws IOException, URISyntaxException { + E621Ripper nextPageRipper = new E621Ripper(new URI("https://e621.net/post/index/1/cosmicminerals").toURL()); try { nextPageRipper.getNextPage(nextPageRipper.getFirstPage()); assert (true); @@ -60,7 +61,7 @@ public class E621RipperTest extends RippersTest { throw e; } - E621Ripper noNextPageRipper = new E621Ripper(new URL("https://e621.net/post/index/1/cosmicminerals")); + E621Ripper noNextPageRipper = new E621Ripper(new URI("https://e621.net/post/index/1/cosmicminerals").toURL()); try { noNextPageRipper.getNextPage(noNextPageRipper.getFirstPage()); } catch (IOException e) { From 1d7d4d764d351c3d9e99bc04c976312c7822179b Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 12 Jun 2023 02:50:51 +0200 Subject: [PATCH 363/512] new URI in App, Http, RipUtils. --- src/main/java/com/rarchives/ripme/App.java | 7 ++-- .../java/com/rarchives/ripme/utils/Http.java | 6 ++- .../com/rarchives/ripme/utils/RipUtils.java | 38 ++++++++++--------- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index 890883ee..2c37fdc9 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -24,6 +24,7 @@ import java.io.BufferedReader; import java.io.FileNotFoundException; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; @@ -183,7 +184,7 @@ public class App { } for (HistoryEntry entry : HISTORY.toList()) { try { - URL url = new URL(entry.url); + URL url = new URI(entry.url).toURL(); rip(url); } catch (Exception e) { logger.error("[!] Failed to rip URL " + entry.url, e); @@ -212,7 +213,7 @@ public class App { if (entry.selected) { added++; try { - URL url = new URL(entry.url); + URL url = new URI(entry.url).toURL(); rip(url); } catch (Exception e) { logger.error("[!] Failed to rip URL " + entry.url, e); @@ -290,7 +291,7 @@ public class App { */ private static void ripURL(String targetURL, boolean saveConfig) { try { - URL url = new URL(targetURL); + URL url = new URI(targetURL).toURL(); rip(url); saveHistory(); } catch (MalformedURLException e) { diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index 35051846..f3b163a0 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -15,6 +15,8 @@ import org.jsoup.nodes.Document; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.HashMap; import java.util.Map; @@ -73,7 +75,7 @@ public class Http { String cookieDomain = ""; try { - URL parsed = new URL(u); + URL parsed = new URI(u).toURL(); String cookieStr = ""; String[] parts = parsed.getHost().split("\\."); @@ -97,7 +99,7 @@ public class Http { if (!cookieStr.equals("")) { cookiesParsed = RipUtils.getCookiesFromString(cookieStr.trim()); } - } catch (MalformedURLException e) { + } catch (MalformedURLException | URISyntaxException e) { logger.warn("Parsing url " + u + " while getting cookies", e); } diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index b001ff01..f0aa0d18 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.utils; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.*; import java.util.regex.Matcher; @@ -60,8 +62,8 @@ public class RipUtils { } else if (url.getHost().endsWith("i.imgur.com") && url.toExternalForm().contains("gifv")) { // links to imgur gifvs try { - result.add(new URL(url.toExternalForm().replaceAll(".gifv", ".mp4"))); - } catch (IOException e) { + result.add(new URI(url.toExternalForm().replaceAll(".gifv", ".mp4")).toURL()); + } catch (IOException | URISyntaxException e) { logger.info("Couldn't get gifv from " + url); } return result; @@ -72,8 +74,8 @@ public class RipUtils { logger.debug("Fetching gfycat page " + url); String videoURL = GfycatRipper.getVideoURL(url); logger.debug("Got gfycat URL: " + videoURL); - result.add(new URL(videoURL)); - } catch (IOException e) { + result.add(new URI(videoURL).toURL()); + } catch (IOException | URISyntaxException e) { // Do nothing logger.warn("Exception while retrieving gfycat page:", e); } @@ -84,8 +86,8 @@ public class RipUtils { logger.debug("Fetching redgifs page " + url); String videoURL = RedgifsRipper.getVideoURL(url); logger.debug("Got redgifs URL: " + videoURL); - result.add(new URL(videoURL)); - } catch (IOException e) { + result.add(new URI(videoURL).toURL()); + } catch (IOException | URISyntaxException e) { // Do nothing logger.warn("Exception while retrieving redgifs page:", e); } @@ -121,9 +123,9 @@ public class RipUtils { EromeRipper r = new EromeRipper(url); Document tempDoc = r.getFirstPage(); for (String u : r.getURLsFromPage(tempDoc)) { - result.add(new URL(u)); + result.add(new URI(u).toURL()); } - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { // Do nothing logger.warn("Exception while retrieving eroshare page:", e); } @@ -135,9 +137,9 @@ public class RipUtils { SoundgasmRipper r = new SoundgasmRipper(url); Document tempDoc = r.getFirstPage(); for (String u : r.getURLsFromPage(tempDoc)) { - result.add(new URL(u)); + result.add(new URI(u).toURL()); } - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { // Do nothing logger.warn("Exception while retrieving soundgasm page:", e); } @@ -150,8 +152,8 @@ public class RipUtils { logger.info("URL: " + url.toExternalForm()); String u = url.toExternalForm().replaceAll("&", "&"); try { - result.add(new URL(u)); - } catch (MalformedURLException e) { + result.add(new URI(u).toURL()); + } catch (MalformedURLException | URISyntaxException e) { } return result; } @@ -161,11 +163,11 @@ public class RipUtils { m = p.matcher(url.toExternalForm()); if (m.matches()) { try { - URL singleURL = new URL(m.group(1)); + URL singleURL = new URI(m.group(1)).toURL(); logger.debug("Found single URL: " + singleURL); result.add(singleURL); return result; - } catch (MalformedURLException e) { + } catch (MalformedURLException | URISyntaxException e) { logger.error("[!] Not a valid URL: '" + url + "'", e); } } @@ -179,19 +181,19 @@ public class RipUtils { .get(); for (Element el : doc.select("meta")) { if (el.attr("property").equals("og:video")) { - result.add(new URL(el.attr("content"))); + result.add(new URI(el.attr("content")).toURL()); return result; } else if (el.attr("name").equals("twitter:image:src")) { - result.add(new URL(el.attr("content"))); + result.add(new URI(el.attr("content")).toURL()); return result; } else if (el.attr("name").equals("twitter:image")) { - result.add(new URL(el.attr("content"))); + result.add(new URI(el.attr("content")).toURL()); return result; } } - } catch (IOException ex) { + } catch (IOException | URISyntaxException ex) { logger.error("[!] Error", ex); } From a9968257356d170e53722b0f710a9234f831d23b Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 12 Jun 2023 03:25:31 +0200 Subject: [PATCH 364/512] new URI in mainwindow. --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 26b3fcaa..357ee6e9 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -37,13 +37,13 @@ import java.io.IOException; import java.io.InputStreamReader; import java.net.MalformedURLException; import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Collections; import java.util.Date; -import java.util.Enumeration; import java.util.List; import java.util.stream.Stream; @@ -738,7 +738,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { if (!urlText.startsWith("http")) { urlText = "http://" + urlText; } - URL url = new URL(urlText); + URL url = new URI(urlText).toURL(); AbstractRipper ripper = AbstractRipper.getRipper(url); statusWithColor(ripper.getHost() + " album detected", Color.GREEN); } catch (Exception e) { @@ -1282,8 +1282,8 @@ public final class MainWindow implements Runnable, RipStatusHandler { } URL url; try { - url = new URL(urlString); - } catch (MalformedURLException e) { + url = new URI(urlString).toURL(); + } catch (MalformedURLException | URISyntaxException e) { LOGGER.error("[!] Could not generate URL for '" + urlString + "'", e); error("Given URL is not valid, expecting http://website.com/page/..."); return null; @@ -1340,7 +1340,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { if (!urlText.startsWith("http")) { urlText = "http://" + urlText; } - URL url = new URL(urlText); + URL url = new URI(urlText).toURL(); // Ripper is needed here to throw/not throw an Exception AbstractRipper ripper = AbstractRipper.getRipper(url); return true; From 45e0e2fa61e8aef4a3148f5b46f44297508b8228 Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 13 Jun 2023 06:19:54 +0200 Subject: [PATCH 365/512] new URI instead of new URL in tests. --- .../ripper/rippers/DeviantartRipperTest.java | 26 ++++++++++--------- .../ripper/rippers/DribbbleRipperTest.java | 7 ++--- .../ripper/rippers/DuckmoviesRipperTest.java | 7 ++--- .../tst/ripper/rippers/EhentaiRipperTest.java | 10 ++++--- 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java index 73982c9f..23dbe679 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DeviantartRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -15,31 +17,31 @@ import org.junit.jupiter.api.Test; public class DeviantartRipperTest extends RippersTest { @Test @Disabled("Broken ripper") - public void testDeviantartAlbum() throws IOException { - DeviantartRipper ripper = new DeviantartRipper(new URL("https://www.deviantart.com/airgee/gallery/")); + public void testDeviantartAlbum() throws IOException, URISyntaxException { + DeviantartRipper ripper = new DeviantartRipper(new URI("https://www.deviantart.com/airgee/gallery/").toURL()); testRipper(ripper); } @Test @Disabled("Broken ripper") - public void testDeviantartNSFWAlbum() throws IOException { + public void testDeviantartNSFWAlbum() throws IOException, URISyntaxException { // NSFW gallery - DeviantartRipper ripper = new DeviantartRipper(new URL("https://www.deviantart.com/faterkcx/gallery/")); + DeviantartRipper ripper = new DeviantartRipper(new URI("https://www.deviantart.com/faterkcx/gallery/").toURL()); testRipper(ripper); } @Test @Disabled("Broken ripper") - public void testGetGID() throws IOException { - URL url = new URL("https://www.deviantart.com/airgee/gallery/"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://www.deviantart.com/airgee/gallery/").toURL(); DeviantartRipper ripper = new DeviantartRipper(url); Assertions.assertEquals("airgee", ripper.getGID(url)); } @Test @Disabled("Broken ripper") - public void testGetGalleryIDAndUsername() throws IOException { - URL url = new URL("https://www.deviantart.com/airgee/gallery/"); + public void testGetGalleryIDAndUsername() throws IOException, URISyntaxException { + URL url = new URI("https://www.deviantart.com/airgee/gallery/").toURL(); DeviantartRipper ripper = new DeviantartRipper(url); Document doc = Http.url(url).get(); // Had to comment because of refactoring/style change @@ -49,11 +51,11 @@ public class DeviantartRipperTest extends RippersTest { @Test @Disabled("Broken ripper") - public void testSanitizeURL() throws IOException { + public void testSanitizeURL() throws IOException, URISyntaxException { List urls = new ArrayList(); - urls.add(new URL("https://www.deviantart.com/airgee/")); - urls.add(new URL("https://www.deviantart.com/airgee")); - urls.add(new URL("https://www.deviantart.com/airgee/gallery/")); + urls.add(new URI("https://www.deviantart.com/airgee/").toURL()); + urls.add(new URI("https://www.deviantart.com/airgee").toURL()); + urls.add(new URI("https://www.deviantart.com/airgee/gallery/").toURL()); for (URL url : urls) { DeviantartRipper ripper = new DeviantartRipper(url); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DribbbleRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DribbbleRipperTest.java index bd4321fc..4a6bf37b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DribbbleRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DribbbleRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.DribbbleRipper; import org.junit.jupiter.api.Disabled; @@ -10,8 +11,8 @@ import org.junit.jupiter.api.Test; public class DribbbleRipperTest extends RippersTest { @Test @Disabled("test or ripper broken") - public void testDribbbleRip() throws IOException { - DribbbleRipper ripper = new DribbbleRipper(new URL("https://dribbble.com/typogriff")); + public void testDribbbleRip() throws IOException, URISyntaxException { + DribbbleRipper ripper = new DribbbleRipper(new URI("https://dribbble.com/typogriff").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DuckmoviesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DuckmoviesRipperTest.java index e4b17cb1..909ddb09 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DuckmoviesRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DuckmoviesRipperTest.java @@ -6,14 +6,15 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; public class DuckmoviesRipperTest extends RippersTest { @Test @Disabled("Broken ripper") - public void testRip() throws IOException { + public void testRip() throws IOException, URISyntaxException { DuckmoviesRipper ripper = new DuckmoviesRipper( - new URL("https://palapaja.com/spyfam-stepbro-gives-in-to-stepsis-asian-persuasion/")); + new URI("https://palapaja.com/spyfam-stepbro-gives-in-to-stepsis-asian-persuasion/").toURL()); testRipper(ripper); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java index 021e892f..31fee74e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.List; @@ -11,15 +13,15 @@ import org.junit.jupiter.api.Test; public class EhentaiRipperTest extends RippersTest { @Test - public void testEHentaiAlbum() throws IOException { - EHentaiRipper ripper = new EHentaiRipper(new URL("https://e-hentai.org/g/1144492/e823bdf9a5/")); + public void testEHentaiAlbum() throws IOException, URISyntaxException { + EHentaiRipper ripper = new EHentaiRipper(new URI("https://e-hentai.org/g/1144492/e823bdf9a5/").toURL()); testRipper(ripper); } // Test the tag black listing @Test - public void testTagBlackList() throws IOException { - URL url = new URL("https://e-hentai.org/g/1228503/1a2f455f96/"); + public void testTagBlackList() throws IOException, URISyntaxException { + URL url = new URI("https://e-hentai.org/g/1228503/1a2f455f96/").toURL(); EHentaiRipper ripper = new EHentaiRipper(url); List tagsOnPage = ripper.getTags(ripper.getFirstPage()); // Test multiple blacklisted tags From 42d9400cac1ecc37796c7f4568a4923f81818ed7 Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 13 Jun 2023 06:27:06 +0200 Subject: [PATCH 366/512] new URI instead of new URL in tests, 2. --- .../rippers/DynastyscansRipperTest.java | 13 ++++---- .../ripper/rippers/EroShareRipperTest.java | 30 ++++++++++--------- .../tst/ripper/rippers/ErofusRipperTest.java | 13 ++++---- .../tst/ripper/rippers/ErotivRipperTest.java | 14 +++++---- .../rippers/FemjoyhunterRipperTest.java | 9 +++--- 5 files changed, 43 insertions(+), 36 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java index fb920545..a2855e98 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DynastyscansRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.DynastyscansRipper; import org.junit.jupiter.api.Assertions; @@ -11,14 +12,14 @@ import org.junit.jupiter.api.Test; public class DynastyscansRipperTest extends RippersTest { @Test @Tag("flaky") - public void testRip() throws IOException { - DynastyscansRipper ripper = new DynastyscansRipper(new URL("https://dynasty-scans.com/chapters/under_one_roof_ch01")); + public void testRip() throws IOException, URISyntaxException { + DynastyscansRipper ripper = new DynastyscansRipper(new URI("https://dynasty-scans.com/chapters/under_one_roof_ch01").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - DynastyscansRipper ripper = new DynastyscansRipper(new URL("https://dynasty-scans.com/chapters/under_one_roof_ch01")); - Assertions.assertEquals("under_one_roof_ch01", ripper.getGID(new URL("https://dynasty-scans.com/chapters/under_one_roof_ch01"))); + public void testGetGID() throws IOException, URISyntaxException { + DynastyscansRipper ripper = new DynastyscansRipper(new URI("https://dynasty-scans.com/chapters/under_one_roof_ch01").toURL()); + Assertions.assertEquals("under_one_roof_ch01", ripper.getGID(new URI("https://dynasty-scans.com/chapters/under_one_roof_ch01").toURL())); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EroShareRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EroShareRipperTest.java index 780460ce..2af1a11a 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EroShareRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EroShareRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.EroShareRipper; @@ -14,46 +16,46 @@ public class EroShareRipperTest extends RippersTest { // single image posts @Test @Disabled("https://github.com/RipMeApp/ripme/issues/306 : EroShareRipper broken (even for eroshae links)") - public void testImageEroshareFromRedditRip() throws IOException { - RedditRipper ripper = new RedditRipper(new URL( - "https://www.reddit.com/r/BestOfEroshare/comments/5z7foo/good_morning_who_likes_abstract_asian_artwork_f/")); + public void testImageEroshareFromRedditRip() throws IOException, URISyntaxException { + RedditRipper ripper = new RedditRipper(new URI( + "https://www.reddit.com/r/BestOfEroshare/comments/5z7foo/good_morning_who_likes_abstract_asian_artwork_f/").toURL()); testRipper(ripper); } @Test @Disabled("https://github.com/RipMeApp/ripme/issues/306 : EroShareRipper broken (even for eroshae links)") - public void testImageEroshareRip() throws IOException { - EroShareRipper ripper = new EroShareRipper(new URL("https://eroshare.com/i/5j2qln3f")); + public void testImageEroshareRip() throws IOException, URISyntaxException { + EroShareRipper ripper = new EroShareRipper(new URI("https://eroshare.com/i/5j2qln3f").toURL()); testRipper(ripper); } @Test @Disabled("https://github.com/RipMeApp/ripme/issues/306 : EroShareRipper broken (even for eroshae links)") - public void testImageEroshaeRip() throws IOException { - EroShareRipper ripper = new EroShareRipper(new URL("https://eroshae.com/i/5j2qln3f")); + public void testImageEroshaeRip() throws IOException, URISyntaxException { + EroShareRipper ripper = new EroShareRipper(new URI("https://eroshae.com/i/5j2qln3f").toURL()); testRipper(ripper); } // video album post @Test @Disabled("https://github.com/RipMeApp/ripme/issues/306 : EroShareRipper broken (even for eroshae links)") - public void testVideoAlbumFromRedditRip() throws IOException { - EroShareRipper ripper = new EroShareRipper(new URL( - "https://www.reddit.com/r/BestOfEroshare/comments/5vyfnw/asian_mf_heard_i_should_post_here_date_night_her/")); + public void testVideoAlbumFromRedditRip() throws IOException, URISyntaxException { + EroShareRipper ripper = new EroShareRipper(new URI( + "https://www.reddit.com/r/BestOfEroshare/comments/5vyfnw/asian_mf_heard_i_should_post_here_date_night_her/").toURL()); testRipper(ripper); } @Test @Disabled("https://github.com/RipMeApp/ripme/issues/306 : EroShareRipper broken (even for eroshae links)") - public void testVideoAlbumEroshareRip() throws IOException { - EroShareRipper ripper = new EroShareRipper(new URL("https://eroshare.com/wqnl6f00")); + public void testVideoAlbumEroshareRip() throws IOException, URISyntaxException { + EroShareRipper ripper = new EroShareRipper(new URI("https://eroshare.com/wqnl6f00").toURL()); testRipper(ripper); } @Test @Disabled("https://github.com/RipMeApp/ripme/issues/306 : EroShareRipper broken (even for eroshae links)") - public void testVideoAlbumEroshaeRip() throws IOException { - EroShareRipper ripper = new EroShareRipper(new URL("https://eroshae.com/wqnl6f00")); + public void testVideoAlbumEroshaeRip() throws IOException, URISyntaxException { + EroShareRipper ripper = new EroShareRipper(new URI("https://eroshae.com/wqnl6f00").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java index 81192040..a06f0e70 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.ErofusRipper; import org.junit.jupiter.api.Assertions; @@ -11,15 +12,15 @@ import org.junit.jupiter.api.Test; public class ErofusRipperTest extends RippersTest { @Test @Tag("flaky") // if url does not exist, erofusripper test ends in out of memory - public void testRip() throws IOException { - ErofusRipper ripper = new ErofusRipper(new URL("https://www.erofus.com/comics/be-story-club-comics/a-kiss/issue-1")); + public void testRip() throws IOException, URISyntaxException { + ErofusRipper ripper = new ErofusRipper(new URI("https://www.erofus.com/comics/be-story-club-comics/a-kiss/issue-1").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testGetGID() throws IOException { - ErofusRipper ripper = new ErofusRipper(new URL("https://www.erofus.com/comics/be-story-club-comics/a-kiss/issue-1")); - Assertions.assertEquals("be-story-club-comics", ripper.getGID(new URL("https://www.erofus.com/comics/be-story-club-comics/a-kiss/issue-1"))); + public void testGetGID() throws IOException, URISyntaxException { + ErofusRipper ripper = new ErofusRipper(new URI("https://www.erofus.com/comics/be-story-club-comics/a-kiss/issue-1").toURL()); + Assertions.assertEquals("be-story-club-comics", ripper.getGID(new URI("https://www.erofus.com/comics/be-story-club-comics/a-kiss/issue-1").toURL())); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErotivRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErotivRipperTest.java index b4afdd67..11f2b59f 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErotivRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErotivRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ErotivRipper; @@ -9,22 +11,22 @@ import org.junit.jupiter.api.Test; public class ErotivRipperTest extends RippersTest { @Test - public void testGetGID() throws IOException { - URL url = new URL("https://erotiv.io/e/1568314255"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://erotiv.io/e/1568314255").toURL(); ErotivRipper ripper = new ErotivRipper(url); assert("1568314255".equals(ripper.getGID(url))); } - public void testRip() throws IOException { - URL url = new URL("https://erotiv.io/e/1568314255"); + public void testRip() throws IOException, URISyntaxException { + URL url = new URI("https://erotiv.io/e/1568314255").toURL(); ErotivRipper ripper = new ErotivRipper(url); testRipper(ripper); } @Test @Disabled("test or ripper broken") - public void testGetURLsFromPage() throws IOException { - URL url = new URL("https://erotiv.io/e/1568314255"); + public void testGetURLsFromPage() throws IOException, URISyntaxException { + URL url = new URI("https://erotiv.io/e/1568314255").toURL(); ErotivRipper ripper = new ErotivRipper(url); assert(1 == ripper.getURLsFromPage(ripper.getFirstPage()).size()); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemjoyhunterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemjoyhunterRipperTest.java index 5520441a..ce1a221f 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemjoyhunterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemjoyhunterRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.FemjoyhunterRipper; @@ -9,9 +10,9 @@ import org.junit.jupiter.api.Test; public class FemjoyhunterRipperTest extends RippersTest { @Test - public void testRip() throws IOException { - FemjoyhunterRipper ripper = new FemjoyhunterRipper(new URL( - "https://www.femjoyhunter.com/alisa-i-got-nice-big-breasts-and-fine-ass-so-she-seems-to-be-a-hottest-brunette-5936/")); + public void testRip() throws IOException, URISyntaxException { + FemjoyhunterRipper ripper = new FemjoyhunterRipper(new URI( + "https://www.femjoyhunter.com/alisa-i-got-nice-big-breasts-and-fine-ass-so-she-seems-to-be-a-hottest-brunette-5936/").toURL()); testRipper(ripper); } } \ No newline at end of file From 854e7f4a51a33408fa23341fb50a6409568205de Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 13 Jun 2023 06:29:36 +0200 Subject: [PATCH 367/512] new URI instead of new URL in tests, 3. --- .../tst/ripper/rippers/FivehundredpxRipperTest.java | 7 ++++--- .../ripme/tst/ripper/rippers/FlickrRipperTest.java | 7 ++++--- .../ripme/tst/ripper/rippers/FolioRipperTest.java | 11 ++++++----- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FivehundredpxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FivehundredpxRipperTest.java index 0392b36e..c4be94d9 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FivehundredpxRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FivehundredpxRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.FivehundredpxRipper; import org.junit.jupiter.api.Disabled; @@ -9,8 +10,8 @@ import org.junit.jupiter.api.Test; public class FivehundredpxRipperTest extends RippersTest { @Test @Disabled("Ripper is broken. See https://github.com/RipMeApp/ripme/issues/438") - public void test500pxAlbum() throws IOException { - FivehundredpxRipper ripper = new FivehundredpxRipper(new URL("https://marketplace.500px.com/alexander_hurman")); + public void test500pxAlbum() throws IOException, URISyntaxException { + FivehundredpxRipper ripper = new FivehundredpxRipper(new URI("https://marketplace.500px.com/alexander_hurman").toURL()); testRipper(ripper); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FlickrRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FlickrRipperTest.java index 02268d64..ea334447 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FlickrRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FlickrRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.FlickrRipper; @@ -11,9 +12,9 @@ import org.junit.jupiter.api.Test; public class FlickrRipperTest extends RippersTest { @Test @Disabled("https://github.com/RipMeApp/ripme/issues/243") - public void testFlickrAlbum() throws IOException { + public void testFlickrAlbum() throws IOException, URISyntaxException { FlickrRipper ripper = new FlickrRipper( - new URL("https://www.flickr.com/photos/leavingallbehind/sets/72157621895942720/")); + new URI("https://www.flickr.com/photos/leavingallbehind/sets/72157621895942720/").toURL()); testRipper(ripper); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java index 9384aebf..cc418842 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.FolioRipper; @@ -11,18 +13,17 @@ import org.junit.jupiter.api.Test; public class FolioRipperTest extends RippersTest { /** * Test for folio.ink ripper - * @throws IOException */ @Test @Disabled("test or ripper broken") - public void testFolioRip() throws IOException { - FolioRipper ripper = new FolioRipper(new URL("https://folio.ink/DmBe6i")); + public void testFolioRip() throws IOException, URISyntaxException { + FolioRipper ripper = new FolioRipper(new URI("https://folio.ink/DmBe6i").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("https://folio.ink/DmBe6i"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://folio.ink/DmBe6i").toURL(); FolioRipper ripper = new FolioRipper(url); Assertions.assertEquals("DmBe6i", ripper.getGID(url)); } From d072dc7ee2dfbb86858ee178fd19a3625d7b6e5c Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 13 Jun 2023 06:35:39 +0200 Subject: [PATCH 368/512] new URI instead of new URL in tests, 4. --- .../tst/ripper/rippers/GirlsOfDesireRipperTest.java | 7 ++++--- .../ripme/tst/ripper/rippers/HbrowseRipperTest.java | 7 ++++--- .../tst/ripper/rippers/Hentai2readRipperTest.java | 7 ++++--- .../tst/ripper/rippers/HentaicafeRipperTest.java | 11 ++++++----- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GirlsOfDesireRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GirlsOfDesireRipperTest.java index 39e6b3c1..59ba2184 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GirlsOfDesireRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GirlsOfDesireRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.GirlsOfDesireRipper; import org.junit.jupiter.api.Tag; @@ -10,8 +11,8 @@ import org.junit.jupiter.api.Test; public class GirlsOfDesireRipperTest extends RippersTest { @Test @Tag("flaky") - public void testGirlsofdesireAlbum() throws IOException { - GirlsOfDesireRipper ripper = new GirlsOfDesireRipper(new URL("http://www.girlsofdesire.org/galleries/krillia/")); + public void testGirlsofdesireAlbum() throws IOException, URISyntaxException { + GirlsOfDesireRipper ripper = new GirlsOfDesireRipper(new URI("http://www.girlsofdesire.org/galleries/krillia/").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HbrowseRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HbrowseRipperTest.java index e4b90c65..291ac782 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HbrowseRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HbrowseRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.HbrowseRipper; import org.junit.jupiter.api.Tag; @@ -10,8 +11,8 @@ import org.junit.jupiter.api.Test; public class HbrowseRipperTest extends RippersTest { @Test @Tag("flaky") - public void testPahealRipper() throws IOException { - HbrowseRipper ripper = new HbrowseRipper(new URL("https://www.hbrowse.com/21013/c00001")); + public void testPahealRipper() throws IOException, URISyntaxException { + HbrowseRipper ripper = new HbrowseRipper(new URI("https://www.hbrowse.com/21013/c00001").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java index c6e2d3de..d6cbb9d0 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.Hentai2readRipper; import org.junit.jupiter.api.Tag; @@ -10,8 +11,8 @@ import org.junit.jupiter.api.Test; public class Hentai2readRipperTest extends RippersTest { @Test @Tag("flaky") - public void testHentai2readAlbum() throws IOException { - Hentai2readRipper ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/1/")); + public void testHentai2readAlbum() throws IOException, URISyntaxException { + Hentai2readRipper ripper = new Hentai2readRipper(new URI("https://hentai2read.com/sm_school_memorial/1/").toURL()); testRipper(ripper); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java index 03a96991..e8c39d07 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.HentaiCafeRipper; import org.junit.jupiter.api.Disabled; @@ -12,16 +13,16 @@ public class HentaicafeRipperTest extends RippersTest { @Test @Tag("flaky") @Disabled("20/05/2021 This test was disabled as the site has experienced notable downtime") - public void testHentaiCafeAlbum() throws IOException { - HentaiCafeRipper ripper = new HentaiCafeRipper(new URL("https://hentai.cafe/kikuta-the-oni-in-the-room/")); + public void testHentaiCafeAlbum() throws IOException, URISyntaxException { + HentaiCafeRipper ripper = new HentaiCafeRipper(new URI("https://hentai.cafe/kikuta-the-oni-in-the-room/").toURL()); testRipper(ripper); } // This album has a line break (
) in the url. Test it to make sure ripme can handle these invalid urls @Test @Tag("flaky") @Disabled("20/05/2021 This test was disabled as the site has experienced notable downtime") - public void testAlbumWithInvalidChars() throws IOException { - HentaiCafeRipper ripper = new HentaiCafeRipper(new URL("https://hentai.cafe/chobipero-club/")); + public void testAlbumWithInvalidChars() throws IOException, URISyntaxException { + HentaiCafeRipper ripper = new HentaiCafeRipper(new URI("https://hentai.cafe/chobipero-club/").toURL()); testRipper(ripper); } From 2bb79a4eacb81426f37bf50011d02174c0c27d05 Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 13 Jun 2023 06:37:47 +0200 Subject: [PATCH 369/512] new URI instead of new URL in tests, Gfycat. --- .../tst/ripper/rippers/GfycatRipperTest.java | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java index 39c14673..ac06ceb6 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java @@ -4,35 +4,33 @@ import com.rarchives.ripme.ripper.rippers.GfycatRipper; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; public class GfycatRipperTest extends RippersTest { /** * Rips correctly formatted URL directly from Gfycat - * @throws IOException */ @Test - public void testGfycatGoodURL() throws IOException{ - GfycatRipper ripper = new GfycatRipper(new URL("https://gfycat.com/TemptingExcellentIchthyosaurs")); + public void testGfycatGoodURL() throws IOException, URISyntaxException { + GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/TemptingExcellentIchthyosaurs").toURL()); testRipper(ripper); } /** * Rips badly formatted URL directly from Gfycat - * @throws IOException */ - public void testGfycatBadURL() throws IOException { - GfycatRipper ripper = new GfycatRipper(new URL("https://gfycat.com/gifs/detail/limitedtestyamericancrow")); + public void testGfycatBadURL() throws IOException, URISyntaxException { + GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/gifs/detail/limitedtestyamericancrow").toURL()); testRipper(ripper); } /** * Rips a Gfycat profile - * @throws IOException */ - public void testGfycatProfile() throws IOException { - GfycatRipper ripper = new GfycatRipper(new URL("https://gfycat.com/@golbanstorage")); + public void testGfycatProfile() throws IOException, URISyntaxException { + GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/@golbanstorage").toURL()); testRipper(ripper); } @@ -40,17 +38,16 @@ public class GfycatRipperTest extends RippersTest { * Rips a Gfycat amp link * @throws IOException */ - public void testGfycatAmp() throws IOException { - GfycatRipper ripper = new GfycatRipper(new URL("https://gfycat.com/amp/TemptingExcellentIchthyosaurs")); + public void testGfycatAmp() throws IOException, URISyntaxException { + GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/amp/TemptingExcellentIchthyosaurs").toURL()); testRipper(ripper); } /** * Rips a Gfycat profile with special characters in username - * @throws IOException */ - public void testGfycatSpecialChar() throws IOException { - GfycatRipper ripper = new GfycatRipper(new URL("https://gfycat.com/@rsss.kr")); + public void testGfycatSpecialChar() throws IOException, URISyntaxException { + GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/@rsss.kr").toURL()); testRipper(ripper); } } From ab8bb3187efac7c5be7f64dde76a6b4e7a364aa9 Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 13 Jun 2023 06:40:57 +0200 Subject: [PATCH 370/512] new URI instead of new URL in tests, 5. --- .../tst/ripper/rippers/FooktubeRipperTest.java | 7 ++++--- .../ripper/rippers/FuraffinityRipperTest.java | 18 ++++++++++-------- .../tst/ripper/rippers/FuskatorRipperTest.java | 11 ++++++----- .../rippers/GfycatporntubeRipperTest.java | 10 ++++++---- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FooktubeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FooktubeRipperTest.java index 10131c19..3e873ed6 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FooktubeRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FooktubeRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.FooktubeRipper; import org.junit.jupiter.api.Disabled; @@ -10,8 +11,8 @@ import org.junit.jupiter.api.Test; public class FooktubeRipperTest extends RippersTest { @Test @Disabled("test or ripper broken") - public void testFooktubeVideo() throws IOException { - FooktubeRipper ripper = new FooktubeRipper(new URL("https://fooktube.com/video/641/in-the-cinema")); //pick any video from the front page + public void testFooktubeVideo() throws IOException, URISyntaxException { + FooktubeRipper ripper = new FooktubeRipper(new URI("https://fooktube.com/video/641/in-the-cinema").toURL()); //pick any video from the front page testRipper(ripper); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java index 8e2e359a..87946d1d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.FuraffinityRipper; @@ -11,26 +13,26 @@ import org.junit.jupiter.api.Test; public class FuraffinityRipperTest extends RippersTest { @Test @Tag("slow") - public void testFuraffinityAlbum() throws IOException { - FuraffinityRipper ripper = new FuraffinityRipper(new URL("https://www.furaffinity.net/gallery/spencerdragon/")); + public void testFuraffinityAlbum() throws IOException, URISyntaxException { + FuraffinityRipper ripper = new FuraffinityRipper(new URI("https://www.furaffinity.net/gallery/spencerdragon/").toURL()); testRipper(ripper); } @Test @Tag("slow") - public void testFuraffinityScrap() throws IOException { - FuraffinityRipper ripper = new FuraffinityRipper(new URL("http://www.furaffinity.net/scraps/sssonic2/")); + public void testFuraffinityScrap() throws IOException, URISyntaxException { + FuraffinityRipper ripper = new FuraffinityRipper(new URI("http://www.furaffinity.net/scraps/sssonic2/").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("https://www.furaffinity.net/gallery/mustardgas/"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://www.furaffinity.net/gallery/mustardgas/").toURL(); FuraffinityRipper ripper = new FuraffinityRipper(url); Assertions.assertEquals("mustardgas", ripper.getGID(url)); } @Test - public void testLogin() throws IOException { - URL url = new URL("https://www.furaffinity.net/gallery/mustardgas/"); + public void testLogin() throws IOException, URISyntaxException { + URL url = new URI("https://www.furaffinity.net/gallery/mustardgas/").toURL(); FuraffinityRipper ripper = new FuraffinityRipper(url); // Check if the first page contain the username of ripmes shared account boolean containsUsername = ripper.getFirstPage().html().contains("ripmethrowaway"); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java index e73f35b4..25656d79 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuskatorRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.FuskatorRipper; import org.junit.jupiter.api.Disabled; @@ -10,14 +11,14 @@ import org.junit.jupiter.api.Test; public class FuskatorRipperTest extends RippersTest { @Test @Disabled("test or ripper broken") - public void testFuskatorAlbum() throws IOException { - FuskatorRipper ripper = new FuskatorRipper(new URL("https://fuskator.com/thumbs/hqt6pPXAf9z/Shaved-Blonde-Babe-Katerina-Ambre.html")); + public void testFuskatorAlbum() throws IOException, URISyntaxException { + FuskatorRipper ripper = new FuskatorRipper(new URI("https://fuskator.com/thumbs/hqt6pPXAf9z/Shaved-Blonde-Babe-Katerina-Ambre.html").toURL()); testRipper(ripper); } @Test @Disabled("test or ripper broken") - public void testUrlsWithTiled() throws IOException { - FuskatorRipper ripper = new FuskatorRipper(new URL("https://fuskator.com/thumbs/hsrzk~UIFmJ/Blonde-Babe-Destiny-Dixon-Playing-With-Black-Dildo.html")); + public void testUrlsWithTiled() throws IOException, URISyntaxException { + FuskatorRipper ripper = new FuskatorRipper(new URI("https://fuskator.com/thumbs/hsrzk~UIFmJ/Blonde-Babe-Destiny-Dixon-Playing-With-Black-Dildo.html").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java index 5b8c4558..ed000e1d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatporntubeRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.GfycatporntubeRipper; @@ -11,14 +13,14 @@ import org.junit.jupiter.api.Test; public class GfycatporntubeRipperTest extends RippersTest { @Test @Tag("flaky") - public void testRip() throws IOException { - GfycatporntubeRipper ripper = new GfycatporntubeRipper(new URL("https://gfycatporntube.com/blowjob-bunny-puts-on-a-show/")); + public void testRip() throws IOException, URISyntaxException { + GfycatporntubeRipper ripper = new GfycatporntubeRipper(new URI("https://gfycatporntube.com/blowjob-bunny-puts-on-a-show/").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("https://gfycatporntube.com/blowjob-bunny-puts-on-a-show/"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://gfycatporntube.com/blowjob-bunny-puts-on-a-show/").toURL(); GfycatporntubeRipper ripper = new GfycatporntubeRipper(url); Assertions.assertEquals("blowjob-bunny-puts-on-a-show", ripper.getGID(url)); } From 7f307ba481e4bab6431471dae23140b3e4c1e1a8 Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 13 Jun 2023 06:50:58 +0200 Subject: [PATCH 371/512] new URI instead of new URL in tests, 6. --- .../ripper/rippers/HentaidudeRipperTest.java | 8 ++--- .../rippers/HentaifoundryRipperTest.java | 17 +++++----- .../ripper/rippers/HentaifoxRipperTest.java | 7 ++-- .../ripper/rippers/HentaiimageRipperTest.java | 7 ++-- .../ripper/rippers/HentainexusRipperTest.java | 12 ++++--- .../tst/ripper/rippers/HitomiRipperTest.java | 9 ++--- .../ripper/rippers/HqpornerRipperTest.java | 34 ++++++++++--------- .../ripper/rippers/HypnohubRipperTest.java | 14 ++++---- .../ripper/rippers/ImagebamRipperTest.java | 7 ++-- .../ripper/rippers/ImagefapRipperTest.java | 12 ++++--- 10 files changed, 70 insertions(+), 57 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaidudeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaidudeRipperTest.java index b8924078..3ac9f4bf 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaidudeRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaidudeRipperTest.java @@ -1,15 +1,15 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.HentaidudeRipper; -import com.rarchives.ripme.utils.Utils; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; public class HentaidudeRipperTest extends RippersTest{ - public void testRip() throws IOException { - HentaidudeRipper ripper = new HentaidudeRipper(new URL("https://hentaidude.com/girlfriends-4ever-dlc-2/")); + public void testRip() throws IOException, URISyntaxException { + HentaidudeRipper ripper = new HentaidudeRipper(new URI("https://hentaidude.com/girlfriends-4ever-dlc-2/").toURL()); testRipper(ripper); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java index 7623c61d..11101bdc 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.HentaifoundryRipper; import org.junit.jupiter.api.Assertions; @@ -9,19 +10,19 @@ import org.junit.jupiter.api.Test; public class HentaifoundryRipperTest extends RippersTest { @Test - public void testHentaifoundryRip() throws IOException { - HentaifoundryRipper ripper = new HentaifoundryRipper(new URL("https://www.hentai-foundry.com/pictures/user/personalami")); + public void testHentaifoundryRip() throws IOException, URISyntaxException { + HentaifoundryRipper ripper = new HentaifoundryRipper(new URI("https://www.hentai-foundry.com/pictures/user/personalami").toURL()); testRipper(ripper); } @Test - public void testHentaifoundryGetGID() throws IOException { - HentaifoundryRipper ripper = new HentaifoundryRipper(new URL("https://www.hentai-foundry.com/stories/user/Rakked")); + public void testHentaifoundryGetGID() throws IOException, URISyntaxException { + HentaifoundryRipper ripper = new HentaifoundryRipper(new URI("https://www.hentai-foundry.com/stories/user/Rakked").toURL()); testRipper(ripper); - Assertions.assertEquals("Rakked", ripper.getGID(new URL("https://www.hentai-foundry.com/stories/user/Rakked"))); + Assertions.assertEquals("Rakked", ripper.getGID(new URI("https://www.hentai-foundry.com/stories/user/Rakked").toURL())); } @Test - public void testHentaifoundryPdfRip() throws IOException { - HentaifoundryRipper ripper = new HentaifoundryRipper(new URL("https://www.hentai-foundry.com/stories/user/Rakked")); + public void testHentaifoundryPdfRip() throws IOException, URISyntaxException { + HentaifoundryRipper ripper = new HentaifoundryRipper(new URI("https://www.hentai-foundry.com/stories/user/Rakked").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoxRipperTest.java index 02515956..edda8162 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoxRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoxRipperTest.java @@ -1,13 +1,14 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.HentaifoxRipper; public class HentaifoxRipperTest extends RippersTest { - public void testRip() throws IOException { - HentaifoxRipper ripper = new HentaifoxRipper(new URL("https://hentaifox.com/gallery/38544/")); + public void testRip() throws IOException, URISyntaxException { + HentaifoxRipper ripper = new HentaifoxRipper(new URI("https://hentaifox.com/gallery/38544/").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaiimageRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaiimageRipperTest.java index 3fab101d..7ccc0029 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaiimageRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaiimageRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.HentaiimageRipper; import com.rarchives.ripme.utils.Utils; @@ -9,9 +10,9 @@ import org.junit.jupiter.api.Test; public class HentaiimageRipperTest extends RippersTest { @Test - public void testHentaifoundryRip() throws IOException { + public void testHentaifoundryRip() throws IOException, URISyntaxException { if (Utils.getConfigBoolean("test.run_flaky_tests", false)) { - HentaiimageRipper ripper = new HentaiimageRipper(new URL("https://hentai-image.com/image/afrobull-gerudo-ongoing-12/")); + HentaiimageRipper ripper = new HentaiimageRipper(new URI("https://hentai-image.com/image/afrobull-gerudo-ongoing-12/").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java index 835f40e6..3389408f 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -17,12 +19,12 @@ public class HentainexusRipperTest extends RippersTest { @Test @Tag("flaky") @Disabled("20/05/2021 This test was disabled as the site has experienced notable downtime") - public void testHentaiNexusJson() throws IOException { + public void testHentaiNexusJson() throws IOException, URISyntaxException { List testURLs = new ArrayList<>(); - testURLs.add(new URL("https://hentainexus.com/view/9202")); - testURLs.add(new URL("https://hentainexus.com/read/9202")); - testURLs.add(new URL("https://hentainexus.com/view/9202#001")); - testURLs.add(new URL("https://hentainexus.com/read/9202#001")); + testURLs.add(new URI("https://hentainexus.com/view/9202").toURL()); + testURLs.add(new URI("https://hentainexus.com/read/9202").toURL()); + testURLs.add(new URI("https://hentainexus.com/view/9202#001").toURL()); + testURLs.add(new URI("https://hentainexus.com/read/9202#001").toURL()); for (URL url : testURLs) { diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java index 646c7f46..5587f773 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HitomiRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.HitomiRipper; @@ -12,9 +13,9 @@ import org.junit.jupiter.api.Test; public class HitomiRipperTest extends RippersTest { @Test @Disabled("Broken ripper") - public void testRip() throws IOException { - HitomiRipper ripper = new HitomiRipper(new URL("https://hitomi.la/galleries/975973.html")); + public void testRip() throws IOException, URISyntaxException { + HitomiRipper ripper = new HitomiRipper(new URI("https://hitomi.la/galleries/975973.html").toURL()); testRipper(ripper); - Assertions.assertTrue(ripper.getGID(new URL("https://hitomi.la/galleries/975973.html")).equals("975973")); + Assertions.assertTrue(ripper.getGID(new URI("https://hitomi.la/galleries/975973.html").toURL()).equals("975973")); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HqpornerRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HqpornerRipperTest.java index c6aebc83..c978f71c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HqpornerRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HqpornerRipperTest.java @@ -6,36 +6,38 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; public class HqpornerRipperTest extends RippersTest { - public void testRip() throws IOException { + public void testRip() throws IOException, URISyntaxException { if (Utils.getConfigBoolean("test.run_flaky_tests", false)) { HqpornerRipper ripper = new HqpornerRipper( - new URL("https://hqporner.com/hdporn/84636-pool_lesson_with_a_cheating_husband.html")); + new URI("https://hqporner.com/hdporn/84636-pool_lesson_with_a_cheating_husband.html").toURL()); testRipper(ripper); } } - public void testGetGID() throws IOException { - URL poolURL = new URL("https://hqporner.com/hdporn/84636-pool_lesson_with_a_cheating_husband.html"); + public void testGetGID() throws IOException, URISyntaxException { + URL poolURL = new URI("https://hqporner.com/hdporn/84636-pool_lesson_with_a_cheating_husband.html").toURL(); HqpornerRipper ripper = new HqpornerRipper(poolURL); Assertions.assertEquals("84636-pool_lesson_with_a_cheating_husband", ripper.getGID(poolURL)); } @Test - public void testGetURLsFromPage() throws IOException { - URL actressUrl = new URL("https://hqporner.com/actress/kali-roses"); + public void testGetURLsFromPage() throws IOException, URISyntaxException { + URL actressUrl = new URI("https://hqporner.com/actress/kali-roses").toURL(); HqpornerRipper ripper = new HqpornerRipper(actressUrl); assert (ripper.getURLsFromPage(ripper.getFirstPage()).size() >= 2); } @Test - public void testGetNextPage() throws IOException { - URL multiPageUrl = new URL("https://hqporner.com/category/tattooed"); + public void testGetNextPage() throws IOException, URISyntaxException { + URL multiPageUrl = new URI("https://hqporner.com/category/tattooed").toURL(); HqpornerRipper multiPageRipper = new HqpornerRipper(multiPageUrl); assert (multiPageRipper.getNextPage(multiPageRipper.getFirstPage()) != null); - URL singlePageUrl = new URL("https://hqporner.com/actress/amy-reid"); + URL singlePageUrl = new URI("https://hqporner.com/actress/amy-reid").toURL(); HqpornerRipper ripper = new HqpornerRipper(singlePageUrl); try { ripper.getNextPage(ripper.getFirstPage()); @@ -44,26 +46,26 @@ public class HqpornerRipperTest extends RippersTest { } } @Test - public void testMyDaddyVideoHost() throws IOException { + public void testMyDaddyVideoHost() throws IOException, URISyntaxException { if (Utils.getConfigBoolean("test.run_flaky_tests", false)) { - URL myDaddyUrl = new URL("https://hqporner.com/hdporn/84636-pool_lesson_with_a_cheating_husband.html"); + URL myDaddyUrl = new URI("https://hqporner.com/hdporn/84636-pool_lesson_with_a_cheating_husband.html").toURL(); HqpornerRipper myDaddyRipper = new HqpornerRipper(myDaddyUrl); testRipper(myDaddyRipper); } } @Test - public void testFlyFlvVideoHost() throws IOException { + public void testFlyFlvVideoHost() throws IOException, URISyntaxException { if (Utils.getConfigBoolean("test.run_flaky_tests", false)) { - URL flyFlvUrl = new URL( - "https://hqporner.com/hdporn/69862-bangbros_-_amy_reid_taking_off_a_tight_sexy_swimsuit.html"); + URL flyFlvUrl = new URI( + "https://hqporner.com/hdporn/69862-bangbros_-_amy_reid_taking_off_a_tight_sexy_swimsuit.html").toURL(); HqpornerRipper flyFlvRipper = new HqpornerRipper(flyFlvUrl); testRipper(flyFlvRipper); } } @Test - public void testUnknownVideoHost() throws IOException { + public void testUnknownVideoHost() throws IOException, URISyntaxException { if (Utils.getConfigBoolean("test.run_flaky_tests", false)) { - URL unknownHostUrl = new URL("https://hqporner.com/hdporn/79528-Kayden_Kross_-_Serious_Masturbation.html"); // howq.cc + URL unknownHostUrl = new URI("https://hqporner.com/hdporn/79528-Kayden_Kross_-_Serious_Masturbation.html").toURL(); // howq.cc HqpornerRipper unknownHostRipper = new HqpornerRipper(unknownHostUrl); testRipper(unknownHostRipper); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java index 001b3d63..46ba5828 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HypnohubRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.HypnohubRipper; @@ -11,21 +13,21 @@ import org.junit.jupiter.api.Test; public class HypnohubRipperTest extends RippersTest { @Test @Disabled("wants a hunman") - public void testRip() throws IOException { - URL poolURL = new URL("http://hypnohub.net/pool/show/2303"); - URL postURL = new URL("http://hypnohub.net/post/show/63464/black_hair-bracelet-collar-corruption-female_only-"); + public void testRip() throws IOException, URISyntaxException { + URL poolURL = new URI("http://hypnohub.net/pool/show/2303").toURL(); + URL postURL = new URI("http://hypnohub.net/post/show/63464/black_hair-bracelet-collar-corruption-female_only-").toURL(); HypnohubRipper ripper = new HypnohubRipper(poolURL); testRipper(ripper); ripper = new HypnohubRipper(postURL); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL poolURL = new URL("http://hypnohub.net/pool/show/2303"); + public void testGetGID() throws IOException, URISyntaxException { + URL poolURL = new URI("http://hypnohub.net/pool/show/2303").toURL(); HypnohubRipper ripper = new HypnohubRipper(poolURL); Assertions.assertEquals("2303", ripper.getGID(poolURL)); - URL postURL = new URL("http://hypnohub.net/post/show/63464/black_hair-bracelet-collar-corruption-female_only-"); + URL postURL = new URI("http://hypnohub.net/post/show/63464/black_hair-bracelet-collar-corruption-female_only-").toURL(); Assertions.assertEquals("63464_black_hair-bracelet-collar-corruption-female_only-", ripper.getGID(postURL)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java index 5ecfe3f6..5f1e9786 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.ImagebamRipper; import org.junit.jupiter.api.Tag; @@ -10,8 +11,8 @@ import org.junit.jupiter.api.Test; public class ImagebamRipperTest extends RippersTest { @Test @Tag("flaky") - public void testImagebamRip() throws IOException { - ImagebamRipper ripper = new ImagebamRipper(new URL("http://www.imagebam.com/gallery/488cc796sllyf7o5srds8kpaz1t4m78i")); + public void testImagebamRip() throws IOException, URISyntaxException { + ImagebamRipper ripper = new ImagebamRipper(new URI("http://www.imagebam.com/gallery/488cc796sllyf7o5srds8kpaz1t4m78i").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java index 19061e34..765eb4ad 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.HashMap; import java.util.Map; @@ -13,15 +15,15 @@ import org.junit.jupiter.api.Test; public class ImagefapRipperTest extends RippersTest { @Test @Tag("flaky") - public void testImagefapAlbums() throws IOException { + public void testImagefapAlbums() throws IOException, URISyntaxException { Map testURLs = new HashMap<>(); // Album with specific title - testURLs.put(new URL("http://www.imagefap.com/pictures/4649440/Frozen-%28Elsa-and-Anna%29?view=2"), + testURLs.put(new URI("http://www.imagefap.com/pictures/4649440/Frozen-%28Elsa-and-Anna%29?view=2").toURL(), "Frozen (Elsa and Anna)"); // New URL format - testURLs.put(new URL("http://www.imagefap.com/gallery.php?pgid=fffd68f659befa5535cf78f014e348f1"), + testURLs.put(new URI("http://www.imagefap.com/gallery.php?pgid=fffd68f659befa5535cf78f014e348f1").toURL(), "imagefap_fffd68f659befa5535cf78f014e348f1"); for (URL url : testURLs.keySet()) { @@ -31,8 +33,8 @@ public class ImagefapRipperTest extends RippersTest { } @Test @Tag("flaky") - public void testImagefapGetAlbumTitle() throws IOException { - URL url = new URL("https://www.imagefap.com/gallery.php?gid=7789753"); + public void testImagefapGetAlbumTitle() throws IOException, URISyntaxException { + URL url = new URI("https://www.imagefap.com/gallery.php?gid=7789753").toURL(); ImagefapRipper ripper = new ImagefapRipper(url); Assertions.assertEquals("imagefap_Red.Heels.Lover.In.Love_7789753", ripper.getAlbumTitle(url)); } From 78e6e7d51e3390dc9c996fa92540f99dd6ea4c04 Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 13 Jun 2023 06:56:27 +0200 Subject: [PATCH 372/512] new URI instead of new URL in tests, 7. --- .../tst/ripper/rippers/ImgboxRipperTest.java | 10 ++-- .../tst/ripper/rippers/ImgurRipperTest.java | 52 ++++++++++--------- .../ripper/rippers/InstagramRipperTest.java | 36 +++++++------ .../ripper/rippers/JagodibujaRipperTest.java | 7 +-- .../ripper/rippers/KingcomixRipperTest.java | 10 ++-- .../tst/ripper/rippers/ListalRipperTest.java | 16 +++--- .../ripper/rippers/LusciousRipperTest.java | 16 +++--- .../ripper/rippers/MangadexRipperTest.java | 11 ++-- .../ripper/rippers/ManganeloRipperTest.java | 10 ++-- 9 files changed, 92 insertions(+), 76 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java index 9e3b6b32..98e3dfc5 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgboxRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ImgboxRipper; @@ -11,14 +13,14 @@ import org.junit.jupiter.api.Test; public class ImgboxRipperTest extends RippersTest { @Test @Tag("flaky") - public void testImgboxRip() throws IOException { - ImgboxRipper ripper = new ImgboxRipper(new URL("https://imgbox.com/g/FJPF7t26FD")); + public void testImgboxRip() throws IOException, URISyntaxException { + ImgboxRipper ripper = new ImgboxRipper(new URI("https://imgbox.com/g/FJPF7t26FD").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("https://imgbox.com/g/FJPF7t26FD"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://imgbox.com/g/FJPF7t26FD").toURL(); ImgboxRipper ripper = new ImgboxRipper(url); Assertions.assertEquals("FJPF7t26FD", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java index 28f18bff..5eeb077f 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java @@ -10,21 +10,23 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; public class ImgurRipperTest extends RippersTest { @Test - public void testImgurURLFailures() throws IOException { + public void testImgurURLFailures() throws IOException, URISyntaxException { List failURLs = new ArrayList<>(); // Imgur urls that should not work - failURLs.add(new URL("http://imgur.com")); - failURLs.add(new URL("http://imgur.com/")); - failURLs.add(new URL("http://i.imgur.com")); - failURLs.add(new URL("http://i.imgur.com/")); - failURLs.add(new URL("http://imgur.com/image.jpg")); - failURLs.add(new URL("http://i.imgur.com/image.jpg")); + failURLs.add(new URI("http://imgur.com").toURL()); + failURLs.add(new URI("http://imgur.com/").toURL()); + failURLs.add(new URI("http://i.imgur.com").toURL()); + failURLs.add(new URI("http://i.imgur.com/").toURL()); + failURLs.add(new URI("http://imgur.com/image.jpg").toURL()); + failURLs.add(new URI("http://i.imgur.com/image.jpg").toURL()); for (URL url : failURLs) { try { new ImgurRipper(url); @@ -37,21 +39,21 @@ public class ImgurRipperTest extends RippersTest { @Test @Tag("flaky") - public void testImgurAlbums() throws IOException { + public void testImgurAlbums() throws IOException, URISyntaxException { List contentURLs = new ArrayList<>(); // URLs that should return more than 1 image - //contentURLs.add(new URL("http://imgur.com/a/dS9OQ#0")); // Horizontal layout - //contentURLs.add(new URL("http://imgur.com/a/YpsW9#0")); // Grid layout - contentURLs.add(new URL("http://imgur.com/a/WxG6f/layout/vertical#0")); - contentURLs.add(new URL("http://imgur.com/a/WxG6f/layout/horizontal#0")); - contentURLs.add(new URL("http://imgur.com/a/WxG6f/layout/grid#0")); - contentURLs.add(new URL("http://imgur.com/gallery/FmP2o")); // Gallery URL + //contentURLs.add(new URI("http://imgur.com/a/dS9OQ#0").toURL()); // Horizontal layout + //contentURLs.add(new URI("http://imgur.com/a/YpsW9#0").toURL()); // Grid layout + contentURLs.add(new URI("http://imgur.com/a/WxG6f/layout/vertical#0").toURL()); + contentURLs.add(new URI("http://imgur.com/a/WxG6f/layout/horizontal#0").toURL()); + contentURLs.add(new URI("http://imgur.com/a/WxG6f/layout/grid#0").toURL()); + contentURLs.add(new URI("http://imgur.com/gallery/FmP2o").toURL()); // Gallery URL // Imgur seems not to support URLs with lists of images anymore. // contentURLs.add(new // URL("http://imgur.com/758qD43,C6iVJex,bP7flAu,J3l85Ri,1U7fhu5,MbuAUCM,JF4vOXQ")); // Sometimes hangs up - // contentURLs.add(new URL("http://imgur.com/r/nsfw_oc/top/all")); - // contentURLs.add(new URL("http://imgur.com/a/bXQpH")); // Album with + // contentURLs.add(new URI("http://imgur.com/r/nsfw_oc/top/all").toURL()); + // contentURLs.add(new URI("http://imgur.com/a/bXQpH").toURL()); // Album with // titles/descriptions for (URL url : contentURLs) { ImgurRipper ripper = new ImgurRipper(url); @@ -61,10 +63,10 @@ public class ImgurRipperTest extends RippersTest { @Test @Disabled("test or ripper broken") - public void testImgurSingleImage() throws IOException { + public void testImgurSingleImage() throws IOException, URISyntaxException { List contentURLs = new ArrayList<>(); - contentURLs.add(new URL("http://imgur.com/qbfcLyG")); // Single image URL - contentURLs.add(new URL("https://imgur.com/KexUO")); // Single image URL + contentURLs.add(new URI("http://imgur.com/qbfcLyG").toURL()); // Single image URL + contentURLs.add(new URI("https://imgur.com/KexUO").toURL()); // Single image URL for (URL url : contentURLs) { ImgurRipper ripper = new ImgurRipper(url); testRipper(ripper); @@ -72,22 +74,22 @@ public class ImgurRipperTest extends RippersTest { } @Test - public void testImgurAlbumWithMoreThan20Pictures() throws IOException { - ImgurAlbum album = ImgurRipper.getImgurAlbum(new URL("http://imgur.com/a/HUMsq")); + public void testImgurAlbumWithMoreThan20Pictures() throws IOException, URISyntaxException { + ImgurAlbum album = ImgurRipper.getImgurAlbum(new URI("http://imgur.com/a/HUMsq").toURL()); Assertions.assertTrue(album.images.size() >= 20, "Failed to find 20 files from " + album.url.toExternalForm() + ", only got " + album.images.size()); } @Test - public void testImgurAlbumWithMoreThan100Pictures() throws IOException { - ImgurAlbum album = ImgurRipper.getImgurAlbum(new URL("https://imgur.com/a/HX3JSrD")); + public void testImgurAlbumWithMoreThan100Pictures() throws IOException, URISyntaxException { + ImgurAlbum album = ImgurRipper.getImgurAlbum(new URI("https://imgur.com/a/HX3JSrD").toURL()); Assertions.assertTrue(album.images.size() >= 100, "Failed to find 100 files from " + album.url.toExternalForm() + ", only got " + album.images.size()); } @Test public void testImgurVideoFromGetFilesFromURL() throws Exception { - List urls = RipUtils.getFilesFromURL(new URL("https://i.imgur.com/4TtwxRN.gifv")); + List urls = RipUtils.getFilesFromURL(new URI("https://i.imgur.com/4TtwxRN.gifv").toURL()); Assertions.assertEquals("https://i.imgur.com/4TtwxRN.mp4", urls.get(0).toExternalForm()); } @@ -97,7 +99,7 @@ public class ImgurRipperTest extends RippersTest { * "over capacity" warning on the page. // I wonder if our testing automation is * what is putting this album over capacity? // See issue #376. public void * testImgurAlbumWithMoreThan1000Pictures() throws IOException { ImgurAlbum - * album = ImgurRipper.getImgurAlbum(new URL("http://imgur.com/a/vsuh5")); + * album = ImgurRipper.getImgurAlbum(new URI("http://imgur.com/a/vsuh5").toURL()); * assertTrue("Failed to find 1000 files from " + album.url.toExternalForm() + * ", only got " + album.images.size(), album.images.size() >= 1000); } */ diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java index c7af1a16..5b929faf 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java @@ -7,6 +7,8 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; @@ -15,19 +17,19 @@ import java.util.Map; public class InstagramRipperTest extends RippersTest { @Test - public void testInstagramGID() throws IOException { + public void testInstagramGID() throws IOException, URISyntaxException { Map testURLs = new HashMap<>(); - testURLs.put(new URL("http://instagram.com/Test_User"), "Test_User"); - testURLs.put(new URL("http://instagram.com/_test_user_"), "_test_user_"); - testURLs.put(new URL("http://instagram.com/_test_user_/?pinned"), "_test_user__pinned"); - testURLs.put(new URL("http://instagram.com/stories/_test_user_/"), "_test_user__stories"); - testURLs.put(new URL("http://instagram.com/_test_user_/tagged"), "_test_user__tagged"); - testURLs.put(new URL("http://instagram.com/_test_user_/channel"), "_test_user__igtv"); - testURLs.put(new URL("http://instagram.com/explore/tags/test_your_tag"), "tag_test_your_tag"); - testURLs.put(new URL("https://www.instagram.com/p/BZ4egP7njW5/?hl=en"), "post_BZ4egP7njW5"); - testURLs.put(new URL("https://www.instagram.com/p/BZ4egP7njW5"), "post_BZ4egP7njW5"); - testURLs.put(new URL("https://www.instagram.com/p/BaNPpaHn2zU/?taken-by=hilaryduff"), "post_BaNPpaHn2zU"); - testURLs.put(new URL("https://www.instagram.com/p/BaNPpaHn2zU/"), "post_BaNPpaHn2zU"); + testURLs.put(new URI("http://instagram.com/Test_User").toURL(), "Test_User"); + testURLs.put(new URI("http://instagram.com/_test_user_").toURL(), "_test_user_"); + testURLs.put(new URI("http://instagram.com/_test_user_/?pinned").toURL(), "_test_user__pinned"); + testURLs.put(new URI("http://instagram.com/stories/_test_user_/").toURL(), "_test_user__stories"); + testURLs.put(new URI("http://instagram.com/_test_user_/tagged").toURL(), "_test_user__tagged"); + testURLs.put(new URI("http://instagram.com/_test_user_/channel").toURL(), "_test_user__igtv"); + testURLs.put(new URI("http://instagram.com/explore/tags/test_your_tag").toURL(), "tag_test_your_tag"); + testURLs.put(new URI("https://www.instagram.com/p/BZ4egP7njW5/?hl=en").toURL(), "post_BZ4egP7njW5"); + testURLs.put(new URI("https://www.instagram.com/p/BZ4egP7njW5").toURL(), "post_BZ4egP7njW5"); + testURLs.put(new URI("https://www.instagram.com/p/BaNPpaHn2zU/?taken-by=hilaryduff").toURL(), "post_BaNPpaHn2zU"); + testURLs.put(new URI("https://www.instagram.com/p/BaNPpaHn2zU/").toURL(), "post_BaNPpaHn2zU"); for (URL url : testURLs.keySet()) { InstagramRipper ripper = new InstagramRipper(url); ripper.setup(); @@ -38,10 +40,10 @@ public class InstagramRipperTest extends RippersTest { @Test @Disabled("Ripper broken for single items") - public void testInstagramSingle() throws IOException { + public void testInstagramSingle() throws IOException, URISyntaxException { List contentURLs = new ArrayList<>(); - contentURLs.add(new URL("https://www.instagram.com/p/BaNPpaHn2zU/?hl=en")); - contentURLs.add(new URL("https://www.instagram.com/p/BaNPpaHn2zU/")); + contentURLs.add(new URI("https://www.instagram.com/p/BaNPpaHn2zU/?hl=en").toURL()); + contentURLs.add(new URI("https://www.instagram.com/p/BaNPpaHn2zU/").toURL()); for (URL url : contentURLs) { InstagramRipper ripper = new InstagramRipper(url); testRipper(ripper); @@ -50,10 +52,10 @@ public class InstagramRipperTest extends RippersTest { @Test @Tag("flaky") - public void testInstagramAlbums() throws IOException { + public void testInstagramAlbums() throws IOException, URISyntaxException { // do not test, in case of rate limit 200/hr since 2021. see // https://github.com/ripmeapp2/ripme/issues/32 - URL url = new URL("https://www.instagram.com/Test_User/"); + URL url = new URI("https://www.instagram.com/Test_User/").toURL(); InstagramRipper ripper = new InstagramRipper(url); testRipper(ripper); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/JagodibujaRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/JagodibujaRipperTest.java index af4314c8..3f5c199e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/JagodibujaRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/JagodibujaRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.JagodibujaRipper; import org.junit.jupiter.api.Disabled; @@ -10,9 +11,9 @@ import org.junit.jupiter.api.Test; public class JagodibujaRipperTest extends RippersTest { @Test @Disabled("fails on github ubuntu automated PR check 2020-07-29") - public void testJagodibujaRipper() throws IOException { + public void testJagodibujaRipper() throws IOException, URISyntaxException { // a photo set - JagodibujaRipper ripper = new JagodibujaRipper(new URL("http://www.jagodibuja.com/comic-in-me/")); + JagodibujaRipper ripper = new JagodibujaRipper(new URI("http://www.jagodibuja.com/comic-in-me/").toURL()); testRipper(ripper); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/KingcomixRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/KingcomixRipperTest.java index bea92e8b..ae543c36 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/KingcomixRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/KingcomixRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.KingcomixRipper; @@ -13,14 +15,14 @@ public class KingcomixRipperTest extends RippersTest { @Test @Disabled("test or ripper broken") - public void testRip() throws IOException { - KingcomixRipper ripper = new KingcomixRipper(new URL("https://kingcomix.com/aunt-cumming-tracy-scops/")); + public void testRip() throws IOException, URISyntaxException { + KingcomixRipper ripper = new KingcomixRipper(new URI("https://kingcomix.com/aunt-cumming-tracy-scops/").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("https://kingcomix.com/aunt-cumming-tracy-scops/"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://kingcomix.com/aunt-cumming-tracy-scops/").toURL(); KingcomixRipper ripper = new KingcomixRipper(url); Assertions.assertEquals("aunt-cumming-tracy-scops", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java index bb4897ba..ca3aee41 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java @@ -1,7 +1,9 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; + import com.rarchives.ripme.ripper.rippers.ListalRipper; import org.junit.jupiter.api.Test; @@ -11,9 +13,9 @@ public class ListalRipperTest extends RippersTest { * Test for list type url. */ @Test - public void testPictures() throws IOException { + public void testPictures() throws IOException, URISyntaxException { ListalRipper ripper = - new ListalRipper(new URL("https://www.listal.com/emma-stone_iii/pictures")); + new ListalRipper(new URI("https://www.listal.com/emma-stone_iii/pictures").toURL()); testRipper(ripper); } @@ -21,9 +23,9 @@ public class ListalRipperTest extends RippersTest { * Test for list type url. */ @Test - public void testRipListType() throws IOException { + public void testRipListType() throws IOException, URISyntaxException { ListalRipper ripper = - new ListalRipper(new URL("https://www.listal.com/list/evolution-emma-stone")); + new ListalRipper(new URI("https://www.listal.com/list/evolution-emma-stone").toURL()); testRipper(ripper); } @@ -31,9 +33,9 @@ public class ListalRipperTest extends RippersTest { * Test for folder type url. */ @Test - public void testRipFolderType() throws IOException { + public void testRipFolderType() throws IOException, URISyntaxException { ListalRipper ripper = - new ListalRipper(new URL("https://www.listal.com/chet-atkins/pictures")); + new ListalRipper(new URI("https://www.listal.com/chet-atkins/pictures").toURL()); testRipper(ripper); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java index 7c797b4b..25dcb94d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.LusciousRipper; @@ -11,28 +13,28 @@ import org.junit.jupiter.api.Test; public class LusciousRipperTest extends RippersTest { @Test @Disabled("test or ripper broken") - public void testPahealRipper() throws IOException { + public void testPahealRipper() throws IOException, URISyntaxException { // a photo set LusciousRipper ripper = new LusciousRipper( - new URL("https://luscious.net/albums/h-na-alice-wa-suki-desu-ka-do-you-like-alice-when_321609/")); + new URI("https://luscious.net/albums/h-na-alice-wa-suki-desu-ka-do-you-like-alice-when_321609/").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("https://luscious.net/albums/h-na-alice-wa-suki-desu-ka-do-you-like-alice-when_321609/"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://luscious.net/albums/h-na-alice-wa-suki-desu-ka-do-you-like-alice-when_321609/").toURL(); LusciousRipper ripper = new LusciousRipper(url); Assertions.assertEquals("h-na-alice-wa-suki-desu-ka-do-you-like-alice-when_321609", ripper.getGID(url)); } @Test @Disabled("test or ripper broken") - public void testGetNextPage() throws IOException { - URL multiPageAlbumUrl = new URL("https://luscious.net/albums/women-of-color_58/"); + public void testGetNextPage() throws IOException, URISyntaxException { + URL multiPageAlbumUrl = new URI("https://luscious.net/albums/women-of-color_58/").toURL(); LusciousRipper multiPageRipper = new LusciousRipper(multiPageAlbumUrl); assert (multiPageRipper.getNextPage(multiPageRipper.getFirstPage()) != null); - URL singlePageAlbumUrl = new URL("https://members.luscious.net/albums/bakaneko-navidarks_332097/"); + URL singlePageAlbumUrl = new URI("https://members.luscious.net/albums/bakaneko-navidarks_332097/").toURL(); LusciousRipper singlePageRipper = new LusciousRipper(singlePageAlbumUrl); try { singlePageRipper.getNextPage(singlePageRipper.getFirstPage()); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MangadexRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MangadexRipperTest.java index 62aabf7c..fe957e32 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MangadexRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MangadexRipperTest.java @@ -4,17 +4,18 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.MangadexRipper; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; public class MangadexRipperTest extends RippersTest{ - public void testRip() throws IOException { - MangadexRipper ripper = new MangadexRipper(new URL("https://mangadex.org/chapter/467904/")); + public void testRip() throws IOException, URISyntaxException { + MangadexRipper ripper = new MangadexRipper(new URI("https://mangadex.org/chapter/467904/").toURL()); testRipper(ripper); } public class testMangaRip extends RippersTest{ - public void testRip() throws IOException { - MangadexRipper ripper = new MangadexRipper(new URL("https://mangadex.org/title/44625/this-croc-will-die-in-100-days")); + public void testRip() throws IOException, URISyntaxException { + MangadexRipper ripper = new MangadexRipper(new URI("https://mangadex.org/title/44625/this-croc-will-die-in-100-days").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java index 37818121..5095553c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ManganeloRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ManganeloRipper; @@ -11,14 +13,14 @@ import org.junit.jupiter.api.Test; public class ManganeloRipperTest extends RippersTest { @Test @Disabled("no images found, test or ripper broken") - public void testRip() throws IOException { - ManganeloRipper ripper = new ManganeloRipper(new URL("https://manganelo.com/manga/demonic_housekeeper")); + public void testRip() throws IOException, URISyntaxException { + ManganeloRipper ripper = new ManganeloRipper(new URI("https://manganelo.com/manga/demonic_housekeeper").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("https://manganelo.com/manga/demonic_housekeeper"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://manganelo.com/manga/demonic_housekeeper").toURL(); ManganeloRipper ripper = new ManganeloRipper(url); Assertions.assertEquals("demonic_housekeeper", ripper.getGID(url)); } From 3c8b67792649a5a9f165dc649a57cd67aacb0950 Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 13 Jun 2023 07:00:21 +0200 Subject: [PATCH 373/512] new URI instead of new URL in tests, 8. --- .../ripper/rippers/EightmusesRipperTest.java | 21 +++++++------- .../ripper/rippers/EroShareRipperTest.java | 1 - .../tst/ripper/rippers/EromeRipperTest.java | 29 ++++++++++--------- .../ripper/rippers/ImagevenueRipperTest.java | 10 ++++--- .../ripper/rippers/MastodonRipperTest.java | 7 +++-- .../ripper/rippers/MastodonXyzRipperTest.java | 7 +++-- .../tst/ripper/rippers/MeituriRipperTest.java | 11 +++---- .../ripper/rippers/ModelmayhemRipperTest.java | 13 +++++---- 8 files changed, 53 insertions(+), 46 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java index 70799d96..2a016b66 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.EightmusesRipper; import org.junit.jupiter.api.Assertions; @@ -11,25 +12,25 @@ import org.junit.jupiter.api.Test; public class EightmusesRipperTest extends RippersTest { @Test @Tag("flaky") - public void testEightmusesAlbum() throws IOException { + public void testEightmusesAlbum() throws IOException, URISyntaxException { // A simple image album - EightmusesRipper ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore")); + EightmusesRipper ripper = new EightmusesRipper(new URI("https://www.8muses.com/comix/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore").toURL()); testRipper(ripper); // Test the new url format - ripper = new EightmusesRipper(new URL("https://www.8muses.com/comics/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore")); + ripper = new EightmusesRipper(new URI("https://www.8muses.com/comics/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore").toURL()); testRipper(ripper); // Test pages with subalbums - ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Blacknwhitecomics_com-Comix/BlacknWhiteComics/The-Mayor")); + ripper = new EightmusesRipper(new URI("https://www.8muses.com/comix/album/Blacknwhitecomics_com-Comix/BlacknWhiteComics/The-Mayor").toURL()); testRipper(ripper); } @Test - public void testGID() throws IOException { - EightmusesRipper ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore")); - Assertions.assertEquals("Affect3D-Comics", ripper.getGID(new URL("https://www.8muses.com/comics/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore"))); + public void testGID() throws IOException, URISyntaxException { + EightmusesRipper ripper = new EightmusesRipper(new URI("https://www.8muses.com/comix/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore").toURL()); + Assertions.assertEquals("Affect3D-Comics", ripper.getGID(new URI("https://www.8muses.com/comics/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore").toURL())); } @Test - public void testGetSubdir() throws IOException { - EightmusesRipper ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore")); + public void testGetSubdir() throws IOException, URISyntaxException { + EightmusesRipper ripper = new EightmusesRipper(new URI("https://www.8muses.com/comix/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore").toURL()); Assertions.assertEquals("After-Party-Issue-1", ripper.getSubdir("After Party - Issue 1")); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EroShareRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EroShareRipperTest.java index 2af1a11a..98d6be8f 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EroShareRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EroShareRipperTest.java @@ -3,7 +3,6 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.net.URL; import com.rarchives.ripme.ripper.rippers.EroShareRipper; import com.rarchives.ripme.ripper.rippers.RedditRipper; diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java index b18762e9..18ddf4bb 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EromeRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.EromeRipper; @@ -10,39 +11,39 @@ import org.junit.jupiter.api.Test; public class EromeRipperTest extends RippersTest { @Test - public void testGetGIDProfilePage() throws IOException { - URL url = new URL("https://www.erome.com/Jay-Jenna"); + public void testGetGIDProfilePage() throws IOException, URISyntaxException { + URL url = new URI("https://www.erome.com/Jay-Jenna").toURL(); EromeRipper ripper = new EromeRipper(url); Assertions.assertEquals("Jay-Jenna", ripper.getGID(url)); } @Test - public void testGetGIDAlbum() throws IOException { - URL url = new URL("https://www.erome.com/a/KbDAM1XT"); + public void testGetGIDAlbum() throws IOException, URISyntaxException { + URL url = new URI("https://www.erome.com/a/KbDAM1XT").toURL(); EromeRipper ripper = new EromeRipper(url); Assertions.assertEquals("KbDAM1XT", ripper.getGID(url)); } @Test - public void testGetAlbumsToQueue() throws IOException { - URL url = new URL("https://www.erome.com/Jay-Jenna"); + public void testGetAlbumsToQueue() throws IOException, URISyntaxException { + URL url = new URI("https://www.erome.com/Jay-Jenna").toURL(); EromeRipper ripper = new EromeRipper(url); assert (2 >= ripper.getAlbumsToQueue(ripper.getFirstPage()).size()); } @Test - public void testPageContainsAlbums() throws IOException { - URL url = new URL("https://www.erome.com/Jay-Jenna"); + public void testPageContainsAlbums() throws IOException, URISyntaxException { + URL url = new URI("https://www.erome.com/Jay-Jenna").toURL(); EromeRipper ripper = new EromeRipper(url); assert (ripper.pageContainsAlbums(url)); - assert (!ripper.pageContainsAlbums(new URL("https://www.erome.com/a/KbDAM1XT"))); + assert (!ripper.pageContainsAlbums(new URI("https://www.erome.com/a/KbDAM1XT").toURL())); } - public void testRip() throws IOException { - URL url = new URL("https://www.erome.com/a/vlefBdsg"); + public void testRip() throws IOException, URISyntaxException { + URL url = new URI("https://www.erome.com/a/vlefBdsg").toURL(); EromeRipper ripper = new EromeRipper(url); testRipper(ripper); } @Test - public void testGetURLsFromPage() throws IOException { - URL url = new URL("https://www.erome.com/a/Tak8F2h6"); + public void testGetURLsFromPage() throws IOException, URISyntaxException { + URL url = new URI("https://www.erome.com/a/Tak8F2h6").toURL(); EromeRipper ripper = new EromeRipper(url); assert (35 == ripper.getURLsFromPage(ripper.getFirstPage()).size()); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagevenueRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagevenueRipperTest.java index 43d211a7..f604d1f7 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagevenueRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagevenueRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ImagevenueRipper; @@ -12,15 +14,15 @@ import org.junit.jupiter.api.Test; public class ImagevenueRipperTest extends RippersTest { @Test @Disabled("See https://github.com/RipMeApp/ripme/issues/1202") - public void testImagevenueRip() throws IOException { + public void testImagevenueRip() throws IOException, URISyntaxException { ImagevenueRipper ripper = new ImagevenueRipper( - new URL("http://img120.imagevenue.com/galshow.php?gal=gallery_1373818527696_191lo")); + new URI("http://img120.imagevenue.com/galshow.php?gal=gallery_1373818527696_191lo").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("http://img120.imagevenue.com/galshow.php?gal=gallery_1373818527696_191lo"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("http://img120.imagevenue.com/galshow.php?gal=gallery_1373818527696_191lo").toURL(); ImagevenueRipper ripper = new ImagevenueRipper(url); Assertions.assertEquals("gallery_1373818527696_191lo", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MastodonRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MastodonRipperTest.java index def7525a..9e50e9a5 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MastodonRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MastodonRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.MastodonRipper; import org.junit.jupiter.api.Tag; @@ -10,8 +11,8 @@ import org.junit.jupiter.api.Test; public class MastodonRipperTest extends RippersTest { @Test @Tag("flaky") - public void testRip() throws IOException { - MastodonRipper ripper = new MastodonRipper(new URL("https://mastodon.social/@pythonhub/media")); + public void testRip() throws IOException, URISyntaxException { + MastodonRipper ripper = new MastodonRipper(new URI("https://mastodon.social/@pythonhub/media").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MastodonXyzRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MastodonXyzRipperTest.java index adbd09c0..0ad1b3f1 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MastodonXyzRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MastodonXyzRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.MastodonXyzRipper; import org.junit.jupiter.api.Tag; @@ -10,8 +11,8 @@ import org.junit.jupiter.api.Test; public class MastodonXyzRipperTest extends RippersTest { @Test @Tag("flaky") - public void testRip() throws IOException { - MastodonXyzRipper ripper = new MastodonXyzRipper(new URL("https://mastodon.xyz/@artwo/media")); + public void testRip() throws IOException, URISyntaxException { + MastodonXyzRipper ripper = new MastodonXyzRipper(new URI("https://mastodon.xyz/@artwo/media").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MeituriRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MeituriRipperTest.java index eb0970fd..441fd5c6 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MeituriRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MeituriRipperTest.java @@ -1,26 +1,27 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.MeituriRipper; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class MeituriRipperTest extends RippersTest { @Test @Tag("flaky") - public void testMeituriRip() throws IOException { - MeituriRipper ripper = new MeituriRipper(new URL("https://www.tujigu.com/a/14449/")); + public void testMeituriRip() throws IOException, URISyntaxException { + MeituriRipper ripper = new MeituriRipper(new URI("https://www.tujigu.com/a/14449/").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("https://www.tujigu.com/a/14449/"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://www.tujigu.com/a/14449/").toURL(); MeituriRipper ripper = new MeituriRipper(url); Assertions.assertEquals("14449", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java index 9e81102a..c8c10ce6 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelmayhemRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.ModelmayhemRipper; @@ -13,16 +14,16 @@ public class ModelmayhemRipperTest extends RippersTest { @Test @Disabled("Broken ripper") - public void testModelmayhemRip() throws IOException { + public void testModelmayhemRip() throws IOException, URISyntaxException { ModelmayhemRipper ripper = new ModelmayhemRipper( - new URL("https://www.modelmayhem.com/portfolio/520206/viewall")); + new URI("https://www.modelmayhem.com/portfolio/520206/viewall").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { + public void testGetGID() throws IOException, URISyntaxException { ModelmayhemRipper ripper = new ModelmayhemRipper( - new URL("https://www.modelmayhem.com/portfolio/520206/viewall")); - Assertions.assertEquals("520206", ripper.getGID(new URL("https://www.modelmayhem.com/portfolio/520206/viewall"))); + new URI("https://www.modelmayhem.com/portfolio/520206/viewall").toURL()); + Assertions.assertEquals("520206", ripper.getGID(new URI("https://www.modelmayhem.com/portfolio/520206/viewall").toURL())); } } From 874c565c56a8fe7c5f6be9740adc65a4d18a6472 Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 13 Jun 2023 07:35:08 +0200 Subject: [PATCH 374/512] new URI instead of new URL in tests, 9. --- .../ripper/rippers/MotherlessRipperTest.java | 7 ++- .../rippers/MyhentaicomicsRipperTest.java | 24 ++++---- .../rippers/MyhentaigalleryRipperTest.java | 10 ++-- .../rippers/MyreadingmangaRipperTest.java | 8 ++- .../ripper/rippers/NatalieMuRipperTest.java | 24 ++++---- .../ripper/rippers/NewgroundsRipperTest.java | 10 ++-- .../tst/ripper/rippers/NfsfwRipperTest.java | 16 ++--- .../tst/ripper/rippers/NhentaiRipperTest.java | 16 ++--- .../tst/ripper/rippers/NsfwXxxRipperTest.java | 7 ++- .../ripper/rippers/NudeGalsRipperTest.java | 13 +++-- .../tst/ripper/rippers/OglafRipperTest.java | 7 ++- .../tst/ripper/rippers/PahealRipperTest.java | 7 ++- .../tst/ripper/rippers/PawooRipperTest.java | 7 ++- .../tst/ripper/rippers/VscoRipperTest.java | 14 +++-- .../ripper/rippers/WebtoonsRipperTest.java | 16 ++--- .../rippers/WordpressComicRipperTest.java | 58 ++++++++++--------- .../tst/ripper/rippers/XcartxRipperTest.java | 7 ++- .../ripper/rippers/XhamsterRipperTest.java | 39 +++++++------ .../tst/ripper/rippers/XlecxRipperTest.java | 7 ++- .../tst/ripper/rippers/XvideosRipperTest.java | 7 ++- .../tst/ripper/rippers/YoupornRipperTest.java | 8 ++- .../tst/ripper/rippers/YuvutuRipperTest.java | 14 +++-- .../tst/ripper/rippers/ZizkiRipperTest.java | 14 +++-- 23 files changed, 187 insertions(+), 153 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java index 97f48a5f..98c65f07 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.MotherlessRipper; @@ -11,8 +12,8 @@ import org.junit.jupiter.api.Test; public class MotherlessRipperTest extends RippersTest { @Test @Tag("flaky") - public void testMotherlessAlbumRip() throws IOException { - MotherlessRipper ripper = new MotherlessRipper(new URL("https://motherless.com/G1168D90")); + public void testMotherlessAlbumRip() throws IOException, URISyntaxException { + MotherlessRipper ripper = new MotherlessRipper(new URI("https://motherless.com/G1168D90").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java index 798176d9..52f8b0d8 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaicomicsRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.MyhentaicomicsRipper; @@ -11,32 +13,32 @@ import org.junit.jupiter.api.Test; public class MyhentaicomicsRipperTest extends RippersTest { @Test @Tag("flaky") - public void testMyhentaicomicsAlbum() throws IOException { - MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(new URL("http://myhentaicomics.com/index.php/Nienna-Lost-Tales")); + public void testMyhentaicomicsAlbum() throws IOException, URISyntaxException { + MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(new URI("http://myhentaicomics.com/index.php/Nienna-Lost-Tales").toURL()); testRipper(ripper); } - public void testGetGID() throws IOException { - URL url = new URL("http://myhentaicomics.com/index.php/Nienna-Lost-Tales"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("http://myhentaicomics.com/index.php/Nienna-Lost-Tales").toURL(); MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(url); // Test a comic Assertions.assertEquals("Nienna-Lost-Tales", ripper.getGID(url)); // Test a search - Assertions.assertEquals("test", ripper.getGID(new URL("http://myhentaicomics.com/index.php/search?q=test"))); + Assertions.assertEquals("test", ripper.getGID(new URI("http://myhentaicomics.com/index.php/search?q=test").toURL())); // Test a tag - Assertions.assertEquals("2409", ripper.getGID(new URL("http://myhentaicomics.com/index.php/tag/2409/"))); + Assertions.assertEquals("2409", ripper.getGID(new URI("http://myhentaicomics.com/index.php/tag/2409/").toURL())); } @Test @Tag("flaky") - public void testGetAlbumsToQueue() throws IOException { - URL url = new URL("https://myhentaicomics.com/index.php/tag/3167/"); + public void testGetAlbumsToQueue() throws IOException, URISyntaxException { + URL url = new URI("https://myhentaicomics.com/index.php/tag/3167/").toURL(); MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(url); Assertions.assertEquals(15, ripper.getAlbumsToQueue(ripper.getFirstPage()).size()); } @Test - public void testPageContainsAlbums() throws IOException { - URL url = new URL("https://myhentaicomics.com/index.php/tag/3167/"); - URL url2 = new URL("https://myhentaicomics.com/index.php/search?q=test"); + public void testPageContainsAlbums() throws IOException, URISyntaxException { + URL url = new URI("https://myhentaicomics.com/index.php/tag/3167/").toURL(); + URL url2 = new URI("https://myhentaicomics.com/index.php/search?q=test").toURL(); MyhentaicomicsRipper ripper = new MyhentaicomicsRipper(url); Assertions.assertTrue(ripper.pageContainsAlbums(url)); Assertions.assertTrue(ripper.pageContainsAlbums(url2)); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaigalleryRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaigalleryRipperTest.java index 54cc9bb2..f7e4273a 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaigalleryRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaigalleryRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.MyhentaigalleryRipper; @@ -11,15 +13,15 @@ import org.junit.jupiter.api.Test; public class MyhentaigalleryRipperTest extends RippersTest { @Test @Tag("flaky") - public void testMyhentaigalleryAlbum() throws IOException { + public void testMyhentaigalleryAlbum() throws IOException, URISyntaxException { MyhentaigalleryRipper ripper = new MyhentaigalleryRipper( - new URL("https://myhentaigallery.com/gallery/thumbnails/9201")); + new URI("https://myhentaigallery.com/gallery/thumbnails/9201").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("https://myhentaigallery.com/gallery/thumbnails/9201"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://myhentaigallery.com/gallery/thumbnails/9201").toURL(); MyhentaigalleryRipper ripper = new MyhentaigalleryRipper(url); Assertions.assertEquals("9201", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyreadingmangaRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyreadingmangaRipperTest.java index 16c5de5e..a6a5a13a 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyreadingmangaRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyreadingmangaRipperTest.java @@ -1,13 +1,15 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; + import com.rarchives.ripme.ripper.rippers.MyreadingmangaRipper; public class MyreadingmangaRipperTest extends RippersTest { - public void testRip() throws IOException { - MyreadingmangaRipper ripper = new MyreadingmangaRipper(new URL("https://myreadingmanga.info/zelo-lee-brave-lover-dj-slave-market-jp/")); + public void testRip() throws IOException, URISyntaxException { + MyreadingmangaRipper ripper = new MyreadingmangaRipper(new URI("https://myreadingmanga.info/zelo-lee-brave-lover-dj-slave-market-jp/").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NatalieMuRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NatalieMuRipperTest.java index 15a9d91a..e3522e71 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NatalieMuRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NatalieMuRipperTest.java @@ -25,10 +25,10 @@ // public void testNatalieMuURLPasses() throws IOException { // List passURLs = new ArrayList<>(); // // URLs that should work -// passURLs.add(new URL("http://natalie.mu/music/news/140367")); -// passURLs.add(new URL("http://cdn2.natalie.mu/music/news/140411")); -// passURLs.add(new URL("http://cdn2.natalie.mu/music/gallery/show/news_id/140411/image_id/369655")); -// passURLs.add(new URL("http://natalie.mu/music/gallery/show/news_id/139146/image_id/365218")); +// passURLs.add(new URI("http://natalie.mu/music/news/140367").toURL()); +// passURLs.add(new URI("http://cdn2.natalie.mu/music/news/140411").toURL()); +// passURLs.add(new URI("http://cdn2.natalie.mu/music/gallery/show/news_id/140411/image_id/369655").toURL()); +// passURLs.add(new URI("http://natalie.mu/music/gallery/show/news_id/139146/image_id/365218").toURL()); // for (URL url : passURLs) { // NatalieMuRipper ripper = new NatalieMuRipper(url); // ripper.setup(); @@ -42,19 +42,19 @@ // public void testNatalieMuRipper() throws IOException { // List contentURLs = new ArrayList<>(); // // URLs that should return more than 1 image -// contentURLs.add(new URL("http://natalie.mu/music/news/140367")); -// contentURLs.add(new URL("http://cdn2.natalie.mu/music/news/140411")); -// contentURLs.add(new URL("http://cdn2.natalie.mu/music/gallery/show/news_id/140411/image_id/369655")); -// contentURLs.add(new URL("http://natalie.mu/music/gallery/show/news_id/139146/image_id/365218")); +// contentURLs.add(new URI("http://natalie.mu/music/news/140367").toURL()); +// contentURLs.add(new URI("http://cdn2.natalie.mu/music/news/140411").toURL()); +// contentURLs.add(new URI("http://cdn2.natalie.mu/music/gallery/show/news_id/140411/image_id/369655").toURL()); +// contentURLs.add(new URI("http://natalie.mu/music/gallery/show/news_id/139146/image_id/365218").toURL()); // // // Most *chans have volatile threads & can't be trusted for integration testing. // -// //contentURLs.add(new URL("http://boards.4chan.org/r/res/12225949")); -// //contentURLs.add(new URL("http://7chan.org/gif/res/23795.html")); -// //contentURLs.add(new URL("http://unichan2.org/b/res/518004.html")); +// //contentURLs.add(new URI("http://boards.4chan.org/r/res/12225949").toURL()); +// //contentURLs.add(new URI("http://7chan.org/gif/res/23795.html").toURL()); +// //contentURLs.add(new URI("http://unichan2.org/b/res/518004.html").toURL()); // // // xchan has an HTTPS certificaiton error... -// //contentURLs.add(new URL("http://xchan.pw/porn/res/437.html")); +// //contentURLs.add(new URI("http://xchan.pw/porn/res/437.html").toURL()); // for (URL url : contentURLs) { // NatalieMuRipper ripper = new NatalieMuRipper(url); // testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NewgroundsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NewgroundsRipperTest.java index 6873c82f..fcfac96d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NewgroundsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NewgroundsRipperTest.java @@ -6,19 +6,21 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; public class NewgroundsRipperTest extends RippersTest { @Test @Tag("flaky") - public void testNewgroundsRip() throws IOException { - NewgroundsRipper ripper = new NewgroundsRipper(new URL("https://zone-sama.newgrounds.com/art")); + public void testNewgroundsRip() throws IOException, URISyntaxException { + NewgroundsRipper ripper = new NewgroundsRipper(new URI("https://zone-sama.newgrounds.com/art").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("https://zone-sama.newgrounds.com/art"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://zone-sama.newgrounds.com/art").toURL(); NewgroundsRipper ripper = new NewgroundsRipper(url); Assertions.assertEquals("zone-sama", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NfsfwRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NfsfwRipperTest.java index 00bba3b7..0477a13d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NfsfwRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NfsfwRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.NfsfwRipper; @@ -13,21 +15,21 @@ public class NfsfwRipperTest extends RippersTest { @Test @Disabled("https://github.com/RipMeApp/ripme/issues/291 -- nfsfw 'account suspended' error; disabled flaky test in CI") - public void testNfsfwRip() throws IOException { - NfsfwRipper ripper = new NfsfwRipper(new URL("http://nfsfw.com/gallery/v/Kitten/")); + public void testNfsfwRip() throws IOException, URISyntaxException { + NfsfwRipper ripper = new NfsfwRipper(new URI("http://nfsfw.com/gallery/v/Kitten/").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("http://nfsfw.com/gallery/v/Kitten/"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("http://nfsfw.com/gallery/v/Kitten/").toURL(); NfsfwRipper ripper = new NfsfwRipper(url); Assertions.assertEquals("Kitten", ripper.getGID(url)); - url = new URL("http://nfsfw.com/gallery/v/Kitten"); + url = new URI("http://nfsfw.com/gallery/v/Kitten").toURL(); Assertions.assertEquals("Kitten", ripper.getGID(url)); - url = new URL("http://nfsfw.com/gallery/v/Kitten/gif_001/"); + url = new URI("http://nfsfw.com/gallery/v/Kitten/gif_001/").toURL(); Assertions.assertEquals("Kitten__gif_001", ripper.getGID(url)); - url = new URL("http://nfsfw.com/gallery/v/Kitten/gif_001/"); + url = new URI("http://nfsfw.com/gallery/v/Kitten/gif_001/").toURL(); Assertions.assertEquals("Kitten__gif_001", ripper.getGID(url)); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java index a1872703..1857e865 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.List; @@ -11,21 +13,21 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class NhentaiRipperTest extends RippersTest { - public void testRip() throws IOException { - NhentaiRipper ripper = new NhentaiRipper(new URL("https://nhentai.net/g/233295/")); + public void testRip() throws IOException, URISyntaxException { + NhentaiRipper ripper = new NhentaiRipper(new URI("https://nhentai.net/g/233295/").toURL()); testRipper(ripper); } - public void testGetGID() throws IOException { - NhentaiRipper ripper = new NhentaiRipper(new URL("https://nhentai.net/g/233295/")); - Assertions.assertEquals("233295", ripper.getGID(new URL("https://nhentai.net/g/233295/"))); + public void testGetGID() throws IOException, URISyntaxException { + NhentaiRipper ripper = new NhentaiRipper(new URI("https://nhentai.net/g/233295/").toURL()); + Assertions.assertEquals("233295", ripper.getGID(new URI("https://nhentai.net/g/233295/").toURL())); } // Test the tag black listing @Test @Tag("flaky") - public void testTagBlackList() throws IOException { - URL url = new URL("https://nhentai.net/g/233295/"); + public void testTagBlackList() throws IOException, URISyntaxException { + URL url = new URI("https://nhentai.net/g/233295/").toURL(); NhentaiRipper ripper = new NhentaiRipper(url); List tagsOnPage = ripper.getTags(ripper.getFirstPage()); // Test multiple blacklisted tags diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NsfwXxxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NsfwXxxRipperTest.java index 029a8541..06e6d5c6 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NsfwXxxRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NsfwXxxRipperTest.java @@ -4,12 +4,13 @@ import com.rarchives.ripme.ripper.rippers.NsfwXxxRipper; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; public class NsfwXxxRipperTest extends RippersTest { @Test - public void testNsfwXxxUser() throws IOException { - NsfwXxxRipper ripper = new NsfwXxxRipper(new URL("https://nsfw.xxx/user/smay3991")); + public void testNsfwXxxUser() throws IOException, URISyntaxException { + NsfwXxxRipper ripper = new NsfwXxxRipper(new URI("https://nsfw.xxx/user/smay3991").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java index 38e697c2..fb348d94 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/NudeGalsRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.NudeGalsRipper; import org.junit.jupiter.api.Assertions; @@ -9,14 +10,14 @@ import org.junit.jupiter.api.Test; public class NudeGalsRipperTest extends RippersTest { @Test - public void testRip() throws IOException { - NudeGalsRipper ripper = new NudeGalsRipper(new URL("https://nude-gals.com/photoshoot.php?photoshoot_id=5541")); + public void testRip() throws IOException, URISyntaxException { + NudeGalsRipper ripper = new NudeGalsRipper(new URI("https://nude-gals.com/photoshoot.php?photoshoot_id=5541").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - NudeGalsRipper ripper = new NudeGalsRipper(new URL("https://nude-gals.com/photoshoot.php?photoshoot_id=5541")); - Assertions.assertEquals("5541", ripper.getGID( new URL("https://nude-gals.com/photoshoot.php?photoshoot_id=5541"))); + public void testGetGID() throws IOException, URISyntaxException { + NudeGalsRipper ripper = new NudeGalsRipper(new URI("https://nude-gals.com/photoshoot.php?photoshoot_id=5541").toURL()); + Assertions.assertEquals("5541", ripper.getGID( new URI("https://nude-gals.com/photoshoot.php?photoshoot_id=5541").toURL())); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/OglafRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/OglafRipperTest.java index 3e716f45..df5eb3dd 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/OglafRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/OglafRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.OglafRipper; @@ -9,8 +10,8 @@ import org.junit.jupiter.api.Test; public class OglafRipperTest extends RippersTest { @Test - public void testRip() throws IOException { - OglafRipper ripper = new OglafRipper(new URL("http://oglaf.com/plumes/")); + public void testRip() throws IOException, URISyntaxException { + OglafRipper ripper = new OglafRipper(new URI("http://oglaf.com/plumes/").toURL()); testRipper(ripper); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PahealRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PahealRipperTest.java index 84403005..d78ad5ef 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PahealRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PahealRipperTest.java @@ -1,16 +1,17 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.PahealRipper; import org.junit.jupiter.api.Test; public class PahealRipperTest extends RippersTest { @Test - public void testPahealRipper() throws IOException { + public void testPahealRipper() throws IOException, URISyntaxException { // a photo set - PahealRipper ripper = new PahealRipper(new URL("http://rule34.paheal.net/post/list/bimbo/1")); + PahealRipper ripper = new PahealRipper(new URI("http://rule34.paheal.net/post/list/bimbo/1").toURL()); testRipper(ripper); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PawooRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PawooRipperTest.java index f2f26b49..664f3fec 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PawooRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PawooRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.PawooRipper; import org.junit.jupiter.api.Tag; @@ -10,8 +11,8 @@ import org.junit.jupiter.api.Test; public class PawooRipperTest extends RippersTest { @Test @Tag("flaky") - public void testRip() throws IOException { - PawooRipper ripper = new PawooRipper(new URL("https://pawoo.net/@halki/media")); + public void testRip() throws IOException, URISyntaxException { + PawooRipper ripper = new PawooRipper(new URI("https://pawoo.net/@halki/media").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java index 2dd58674..aa963589 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java @@ -7,6 +7,8 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; public class VscoRipperTest extends RippersTest { @@ -18,8 +20,8 @@ public class VscoRipperTest extends RippersTest { */ @Test @Tag("flaky") - public void testSingleImageRip() throws IOException { - VscoRipper ripper = new VscoRipper(new URL("https://vsco.co/jonathangodoy/media/5d1aec76bb669a128035e98a")); + public void testSingleImageRip() throws IOException, URISyntaxException { + VscoRipper ripper = new VscoRipper(new URI("https://vsco.co/jonathangodoy/media/5d1aec76bb669a128035e98a").toURL()); testRipper(ripper); } @@ -30,8 +32,8 @@ public class VscoRipperTest extends RippersTest { * @throws IOException */ @Test - public void testHyphenatedRip() throws IOException { - VscoRipper ripper = new VscoRipper(new URL("https://vsco.co/jolly-roger/gallery")); + public void testHyphenatedRip() throws IOException, URISyntaxException { + VscoRipper ripper = new VscoRipper(new URI("https://vsco.co/jolly-roger/gallery").toURL()); testRipper(ripper); } @@ -41,8 +43,8 @@ public class VscoRipperTest extends RippersTest { * @throws IOException */ @Test - public void testGetGID() throws IOException { - URL url = new URL("https://vsco.co/jolly-roger/media/590359c4ade3041f2658f407"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://vsco.co/jolly-roger/media/590359c4ade3041f2658f407").toURL(); VscoRipper ripper = new VscoRipper(url); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java index 500f507e..d05f307c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WebtoonsRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.WebtoonsRipper; @@ -9,23 +11,23 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -public class WebtoonsRipperTest extends RippersTest { +public class WebtoonsRipperTest extends RippersTest { @Test @Tag("flaky") - public void testWebtoonsAlbum() throws IOException { - WebtoonsRipper ripper = new WebtoonsRipper(new URL("https://www.webtoons.com/en/super-hero/unordinary/episode-103/viewer?title_no=679&episode_no=109")); + public void testWebtoonsAlbum() throws IOException, URISyntaxException { + WebtoonsRipper ripper = new WebtoonsRipper(new URI("https://www.webtoons.com/en/super-hero/unordinary/episode-103/viewer?title_no=679&episode_no=109").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testWedramabtoonsType() throws IOException { - WebtoonsRipper ripper = new WebtoonsRipper(new URL("http://www.webtoons.com/en/drama/lookism/ep-145/viewer?title_no=1049&episode_no=145")); + public void testWedramabtoonsType() throws IOException, URISyntaxException { + WebtoonsRipper ripper = new WebtoonsRipper(new URI("http://www.webtoons.com/en/drama/lookism/ep-145/viewer?title_no=1049&episode_no=145").toURL()); testRipper(ripper); } @Test @Disabled("URL format different") - public void testGetGID() throws IOException { - URL url = new URL("https://www.webtoons.com/en/super-hero/unordinary/episode-103/viewer?title_no=679&episode_no=109"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://www.webtoons.com/en/super-hero/unordinary/episode-103/viewer?title_no=679&episode_no=109").toURL(); WebtoonsRipper ripper = new WebtoonsRipper(url); Assertions.assertEquals("super-hero", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java index e8f21726..b27234f9 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.WordpressComicRipper; @@ -24,50 +26,50 @@ public class WordpressComicRipperTest extends RippersTest { @Test @Tag("flaky") // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI - public void test_totempole666() throws IOException { + public void test_totempole666() throws IOException, URISyntaxException { WordpressComicRipper ripper = new WordpressComicRipper( - new URL("http://www.totempole666.com/comic/first-time-for-everything-00-cover/")); + new URI("http://www.totempole666.com/comic/first-time-for-everything-00-cover/").toURL()); testRipper(ripper); } @Test @Tag("flaky") // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI - public void test_buttsmithy() throws IOException { - WordpressComicRipper ripper = new WordpressComicRipper(new URL("http://buttsmithy.com/archives/comic/p1")); + public void test_buttsmithy() throws IOException, URISyntaxException { + WordpressComicRipper ripper = new WordpressComicRipper(new URI("http://buttsmithy.com/archives/comic/p1").toURL()); testRipper(ripper); } @Test @Tag("flaky") // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI - public void test_themonsterunderthebed() throws IOException { + public void test_themonsterunderthebed() throws IOException, URISyntaxException { WordpressComicRipper ripper = new WordpressComicRipper( - new URL("http://themonsterunderthebed.net/?comic=test-post")); + new URI("http://themonsterunderthebed.net/?comic=test-post").toURL()); testRipper(ripper); } @Test - public void test_prismblush() throws IOException { + public void test_prismblush() throws IOException, URISyntaxException { WordpressComicRipper ripper = new WordpressComicRipper( - new URL("http://prismblush.com/comic/hella-trap-pg-01/")); + new URI("http://prismblush.com/comic/hella-trap-pg-01/").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void test_konradokonski_1() throws IOException { + public void test_konradokonski_1() throws IOException, URISyntaxException { WordpressComicRipper ripper = new WordpressComicRipper( - new URL("http://www.konradokonski.com/sawdust/comic/get-up/")); + new URI("http://www.konradokonski.com/sawdust/comic/get-up/").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void test_konradokonski_2() throws IOException { + public void test_konradokonski_2() throws IOException, URISyntaxException { WordpressComicRipper ripper = new WordpressComicRipper( - new URL("http://www.konradokonski.com/wiory/comic/08182008/")); + new URI("http://www.konradokonski.com/wiory/comic/08182008/").toURL()); testRipper(ripper); } @Test - public void test_konradokonski_getAlbumTitle() throws IOException { - URL url = new URL("http://www.konradokonski.com/sawdust/comic/get-up/"); + public void test_konradokonski_getAlbumTitle() throws IOException, URISyntaxException { + URL url = new URI("http://www.konradokonski.com/sawdust/comic/get-up/").toURL(); WordpressComicRipper ripper = new WordpressComicRipper(url); Assertions.assertEquals("konradokonski.com_sawdust", ripper.getAlbumTitle(url)); @@ -75,47 +77,47 @@ public class WordpressComicRipperTest extends RippersTest { @Test @Tag("flaky") // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI - public void test_freeadultcomix() throws IOException { + public void test_freeadultcomix() throws IOException, URISyntaxException { WordpressComicRipper ripper = new WordpressComicRipper( - new URL("http://freeadultcomix.com/finders-feepaid-in-full-sparrow/")); + new URI("http://freeadultcomix.com/finders-feepaid-in-full-sparrow/").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void test_delvecomic() throws IOException { + public void test_delvecomic() throws IOException, URISyntaxException { WordpressComicRipper ripper = new WordpressComicRipper( - new URL("http://thisis.delvecomic.com/NewWP/comic/in-too-deep/")); + new URI("http://thisis.delvecomic.com/NewWP/comic/in-too-deep/").toURL()); testRipper(ripper); } @Test - public void test_Eightmuses_download() throws IOException { + public void test_Eightmuses_download() throws IOException, URISyntaxException { WordpressComicRipper ripper = new WordpressComicRipper( - new URL("https://8muses.download/lustomic-playkittens-josh-samuel-porn-comics-8-muses/")); + new URI("https://8muses.download/lustomic-playkittens-josh-samuel-porn-comics-8-muses/").toURL()); testRipper(ripper); } @Test - public void test_Eightmuses_getAlbumTitle() throws IOException { - URL url = new URL("https://8muses.download/lustomic-playkittens-josh-samuel-porn-comics-8-muses/"); + public void test_Eightmuses_getAlbumTitle() throws IOException, URISyntaxException { + URL url = new URI("https://8muses.download/lustomic-playkittens-josh-samuel-porn-comics-8-muses/").toURL(); WordpressComicRipper ripper = new WordpressComicRipper(url); Assertions.assertEquals("8muses.download_lustomic-playkittens-josh-samuel-porn-comics-8-muses", ripper.getAlbumTitle(url)); } @Test - public void test_spyingwithlana_download() throws IOException { + public void test_spyingwithlana_download() throws IOException, URISyntaxException { WordpressComicRipper ripper = new WordpressComicRipper( - new URL("http://spyingwithlana.com/comic/the-big-hookup/")); + new URI("http://spyingwithlana.com/comic/the-big-hookup/").toURL()); testRipper(ripper); } @Test - public void test_spyingwithlana_getAlbumTitle() throws IOException { - URL url = new URL("http://spyingwithlana.com/comic/the-big-hookup/"); + public void test_spyingwithlana_getAlbumTitle() throws IOException, URISyntaxException { + URL url = new URI("http://spyingwithlana.com/comic/the-big-hookup/").toURL(); WordpressComicRipper ripper = new WordpressComicRipper(url); Assertions.assertEquals("spyingwithlana_the-big-hookup", ripper.getAlbumTitle(url)); } @Test @Tag("flaky") // https://github.com/RipMeApp/ripme/issues/269 - Disabled test - WordpressRipperTest: various domains flaky in CI - public void test_pepsaga() throws IOException { - WordpressComicRipper ripper = new WordpressComicRipper(new URL("http://shipinbottle.pepsaga.com/?p=281")); + public void test_pepsaga() throws IOException, URISyntaxException { + WordpressComicRipper ripper = new WordpressComicRipper(new URI("http://shipinbottle.pepsaga.com/?p=281").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XcartxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XcartxRipperTest.java index 7b5ab870..b4130cbb 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XcartxRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XcartxRipperTest.java @@ -6,13 +6,14 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; public class XcartxRipperTest extends RippersTest { @Test @Disabled("Broken ripper") - public void testAlbum() throws IOException { - XcartxRipper ripper = new XcartxRipper(new URL("http://xcartx.com/4937-tokimeki-nioi.html")); + public void testAlbum() throws IOException, URISyntaxException { + XcartxRipper ripper = new XcartxRipper(new URI("http://xcartx.com/4937-tokimeki-nioi.html").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java index 1f1af488..68747ea4 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java @@ -1,12 +1,13 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.XhamsterRipper; import org.jsoup.nodes.Document; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -14,56 +15,56 @@ import org.junit.jupiter.api.Test; public class XhamsterRipperTest extends RippersTest { @Test @Tag("flaky") - public void testXhamsterAlbum1() throws IOException { - XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/photos/gallery/sexy-preggo-girls-9026608")); + public void testXhamsterAlbum1() throws IOException, URISyntaxException { + XhamsterRipper ripper = new XhamsterRipper(new URI("https://xhamster.com/photos/gallery/sexy-preggo-girls-9026608").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testXhamster2Album() throws IOException { - XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster2.com/photos/gallery/sexy-preggo-girls-9026608")); + public void testXhamster2Album() throws IOException, URISyntaxException { + XhamsterRipper ripper = new XhamsterRipper(new URI("https://xhamster2.com/photos/gallery/sexy-preggo-girls-9026608").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testXhamsterAlbum2() throws IOException { - XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664")); + public void testXhamsterAlbum2() throws IOException, URISyntaxException { + XhamsterRipper ripper = new XhamsterRipper(new URI("https://xhamster.com/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testXhamsterAlbumOneDomain() throws IOException { - XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.one/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664")); + public void testXhamsterAlbumOneDomain() throws IOException, URISyntaxException { + XhamsterRipper ripper = new XhamsterRipper(new URI("https://xhamster.one/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testXhamsterAlbumDesiDomain() throws IOException { - XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster5.desi/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664")); + public void testXhamsterAlbumDesiDomain() throws IOException, URISyntaxException { + XhamsterRipper ripper = new XhamsterRipper(new URI("https://xhamster5.desi/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testXhamsterVideo() throws IOException { - XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/videos/brazzers-busty-big-booty-milf-lisa-ann-fucks-her-masseur-1492828")); + public void testXhamsterVideo() throws IOException, URISyntaxException { + XhamsterRipper ripper = new XhamsterRipper(new URI("https://xhamster.com/videos/brazzers-busty-big-booty-milf-lisa-ann-fucks-her-masseur-1492828").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testBrazilianXhamster() throws IOException { - XhamsterRipper ripper = new XhamsterRipper(new URL("https://pt.xhamster.com/photos/gallery/cartoon-babe-15786301")); + public void testBrazilianXhamster() throws IOException, URISyntaxException { + XhamsterRipper ripper = new XhamsterRipper(new URI("https://pt.xhamster.com/photos/gallery/cartoon-babe-15786301").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("https://xhamster5.desi/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://xhamster5.desi/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664").toURL(); XhamsterRipper ripper = new XhamsterRipper(url); Assertions.assertEquals("7254664", ripper.getGID(url)); } @Test @Tag("flaky") - public void testGetNextPage() throws IOException { - XhamsterRipper ripper = new XhamsterRipper(new URL("https://pt.xhamster.com/photos/gallery/mega-compil-6-10728626")); + public void testGetNextPage() throws IOException, URISyntaxException { + XhamsterRipper ripper = new XhamsterRipper(new URI("https://pt.xhamster.com/photos/gallery/mega-compil-6-10728626").toURL()); Document doc = ripper.getFirstPage(); try { ripper.getNextPage(doc); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XlecxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XlecxRipperTest.java index 807231e8..2b943c7d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XlecxRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XlecxRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.XlecxRipper; @@ -11,8 +12,8 @@ import org.junit.jupiter.api.Test; public class XlecxRipperTest extends RippersTest { @Test @Disabled("Broken ripper") - public void testAlbum() throws IOException { - XlecxRipper ripper = new XlecxRipper(new URL("http://xlecx.com/4274-black-canary-ravished-prey.html")); + public void testAlbum() throws IOException, URISyntaxException { + XlecxRipper ripper = new XlecxRipper(new URI("http://xlecx.com/4274-black-canary-ravished-prey.html").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XvideosRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XvideosRipperTest.java index 3c76c362..cde9d111 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XvideosRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XvideosRipperTest.java @@ -1,15 +1,16 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.XvideosRipper; import org.junit.jupiter.api.Test; public class XvideosRipperTest extends RippersTest { @Test - public void testXhamsterAlbum1() throws IOException { - XvideosRipper ripper = new XvideosRipper(new URL("https://www.xvideos.com/video23515878/dee_s_pool_toys")); + public void testXhamsterAlbum1() throws IOException, URISyntaxException { + XvideosRipper ripper = new XvideosRipper(new URI("https://www.xvideos.com/video23515878/dee_s_pool_toys").toURL()); testRipper(ripper); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YoupornRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YoupornRipperTest.java index 68bf2b69..9520ee08 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YoupornRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YoupornRipperTest.java @@ -5,6 +5,8 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -12,11 +14,11 @@ import java.util.List; public class YoupornRipperTest extends RippersTest { @Test @Tag("flaky") - public void testYoupornRipper() throws IOException { + public void testYoupornRipper() throws IOException, URISyntaxException { List contentURLs = new ArrayList<>(); // Video cannot be loaded: "Video has been flagged for verification" - //contentURLs.add(new URL("http://www.youporn.com/watch/7669155/mrs-li-amateur-69-orgasm/?from=categ")); - contentURLs.add(new URL("https://www.youporn.com/watch/13158849/smashing-star-slut-part-2/")); + //contentURLs.add(new URI("http://www.youporn.com/watch/7669155/mrs-li-amateur-69-orgasm/?from=categ").toURL()); + contentURLs.add(new URI("https://www.youporn.com/watch/13158849/smashing-star-slut-part-2/").toURL()); for (URL url : contentURLs) { YoupornRipper ripper = new YoupornRipper(url); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java index ec95a02c..d8c8d756 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.YuvutuRipper; @@ -9,18 +11,18 @@ import org.junit.jupiter.api.Test; public class YuvutuRipperTest extends RippersTest { @Test - public void testYuvutuAlbum1() throws IOException { - YuvutuRipper ripper = new YuvutuRipper(new URL("http://www.yuvutu.com/modules.php?name=YuGallery&action=view&set_id=127013")); + public void testYuvutuAlbum1() throws IOException, URISyntaxException { + YuvutuRipper ripper = new YuvutuRipper(new URI("http://www.yuvutu.com/modules.php?name=YuGallery&action=view&set_id=127013").toURL()); testRipper(ripper); } @Test - public void testYuvutuAlbum2() throws IOException { - YuvutuRipper ripper = new YuvutuRipper(new URL("http://www.yuvutu.com/modules.php?name=YuGallery&action=view&set_id=420333")); + public void testYuvutuAlbum2() throws IOException, URISyntaxException { + YuvutuRipper ripper = new YuvutuRipper(new URI("http://www.yuvutu.com/modules.php?name=YuGallery&action=view&set_id=420333").toURL()); testRipper(ripper); } - public void testGetGID() throws IOException { - URL url = new URL("http://www.yuvutu.com/modules.php?name=YuGallery&action=view&set_id=420333"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("http://www.yuvutu.com/modules.php?name=YuGallery&action=view&set_id=420333").toURL(); YuvutuRipper ripper = new YuvutuRipper(url); Assertions.assertEquals("420333", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java index 3d21df97..adbd4c77 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ZizkiRipper; @@ -12,22 +14,22 @@ public class ZizkiRipperTest extends RippersTest { @Test @Tag("flaky") - public void testRip() throws IOException { - ZizkiRipper ripper = new ZizkiRipper(new URL("http://zizki.com/dee-chorde/we-got-spirit")); + public void testRip() throws IOException, URISyntaxException { + ZizkiRipper ripper = new ZizkiRipper(new URI("http://zizki.com/dee-chorde/we-got-spirit").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("http://zizki.com/dee-chorde/we-got-spirit"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("http://zizki.com/dee-chorde/we-got-spirit").toURL(); ZizkiRipper ripper = new ZizkiRipper(url); Assertions.assertEquals("dee-chorde", ripper.getGID(url)); } @Test @Tag("flaky") - public void testAlbumTitle() throws IOException { - URL url = new URL("http://zizki.com/dee-chorde/we-got-spirit"); + public void testAlbumTitle() throws IOException, URISyntaxException { + URL url = new URI("http://zizki.com/dee-chorde/we-got-spirit").toURL(); ZizkiRipper ripper = new ZizkiRipper(url); Assertions.assertEquals("zizki_Dee Chorde_We Got Spirit", ripper.getAlbumTitle(url)); } From 589d7eb5a501914be6701d875fcd05458e3422f3 Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 13 Jun 2023 07:43:41 +0200 Subject: [PATCH 375/512] new URI instead of new URL in tests, 10. --- .../tst/ripper/rippers/PornhubRipperTest.java | 16 ++++---- .../tst/ripper/rippers/RedditRipperTest.java | 39 ++++++++++--------- .../tst/ripper/rippers/RedgifsRipperTest.java | 23 +++++------ .../tst/ripper/rippers/Rule34RipperTest.java | 10 +++-- .../ripper/rippers/RulePornRipperTest.java | 10 +++-- .../rippers/SankakuComplexRipperTest.java | 18 +++++---- .../ripper/rippers/ScrolllerRipperTest.java | 34 ++++++++-------- .../ripper/rippers/ShesFreakyRipperTest.java | 10 +++-- .../ripper/rippers/TapasticRipperTest.java | 10 +++-- 9 files changed, 93 insertions(+), 77 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java index 22de8349..1bc6520f 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.PornhubRipper; @@ -13,26 +15,26 @@ import org.junit.jupiter.api.Test; public class PornhubRipperTest extends RippersTest { @Test - public void testPornhubRip() throws IOException { + public void testPornhubRip() throws IOException, URISyntaxException { if (Utils.getConfigBoolean("test.run_flaky_tests", false)) { - PornhubRipper ripper = new PornhubRipper(new URL("https://www.pornhub.com/album/15680522")); + PornhubRipper ripper = new PornhubRipper(new URI("https://www.pornhub.com/album/15680522").toURL()); testRipper(ripper); } } - public void testGetGID() throws IOException { - URL url = new URL("https://www.pornhub.com/album/15680522?page=2"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://www.pornhub.com/album/15680522?page=2").toURL(); PornhubRipper ripper = new PornhubRipper(url); Assertions.assertEquals("15680522", ripper.getGID(url)); - url = new URL("https://www.pornhub.com/album/15680522"); + url = new URI("https://www.pornhub.com/album/15680522").toURL(); Assertions.assertEquals("15680522", ripper.getGID(url)); } @Test @Tag("flaky") - public void testGetNextPage() throws IOException { + public void testGetNextPage() throws IOException, URISyntaxException { String baseURL = "https://www.pornhub.com/album/30687901"; - PornhubRipper ripper = new PornhubRipper(new URL(baseURL)); + PornhubRipper ripper = new PornhubRipper(new URI(baseURL).toURL()); Document page = Http.url(baseURL).get(); int numPagesRemaining = 1; for (int idx = 0; idx < numPagesRemaining; idx++){ diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java index b6eae52b..db0fa530 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java @@ -2,7 +2,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.RedditRipper; @@ -14,23 +15,23 @@ public class RedditRipperTest extends RippersTest { @Test @Tag("flaky") // https://github.com/RipMeApp/ripme/issues/253 - public void testRedditSubredditRip() throws IOException { - RedditRipper ripper = new RedditRipper(new URL("http://www.reddit.com/r/nsfw_oc")); + public void testRedditSubredditRip() throws IOException, URISyntaxException { + RedditRipper ripper = new RedditRipper(new URI("http://www.reddit.com/r/nsfw_oc").toURL()); testRipper(ripper); } @Test @Tag("flaky") // https://github.com/RipMeApp/ripme/issues/253 - public void testRedditSubredditTopRip() throws IOException { - RedditRipper ripper = new RedditRipper(new URL("http://www.reddit.com/r/nsfw_oc/top?t=all")); + public void testRedditSubredditTopRip() throws IOException, URISyntaxException { + RedditRipper ripper = new RedditRipper(new URI("http://www.reddit.com/r/nsfw_oc/top?t=all").toURL()); testRipper(ripper); } @Test @Disabled - public void testRedditPostRip() throws IOException { + public void testRedditPostRip() throws IOException, URISyntaxException { RedditRipper ripper = new RedditRipper( - new URL("http://www.reddit.com/r/UnrealGirls/comments/1ziuhl/in_class_veronique_popa/")); + new URI("http://www.reddit.com/r/UnrealGirls/comments/1ziuhl/in_class_veronique_popa/").toURL()); testRipper(ripper); } @@ -41,25 +42,25 @@ public class RedditRipperTest extends RippersTest { */ @Test @Tag("flaky") - public void testRedditGfyGoodURL() throws IOException { + public void testRedditGfyGoodURL() throws IOException, URISyntaxException { RedditRipper ripper = new RedditRipper( - new URL("https://www.reddit.com/r/bottesting/comments/7msozf/good_link/")); + new URI("https://www.reddit.com/r/bottesting/comments/7msozf/good_link/").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testSelfPostRip() throws IOException { + public void testSelfPostRip() throws IOException, URISyntaxException { RedditRipper ripper = new RedditRipper( - new URL("https://www.reddit.com/r/gonewildstories/comments/oz7d97/f_18_finally_having_a_normal_sex_life/") + new URI("https://www.reddit.com/r/gonewildstories/comments/oz7d97/f_18_finally_having_a_normal_sex_life/").toURL() ); testRipper(ripper); } @Test @Tag("flaky") - public void testSelfPostAuthorRip() throws IOException { - RedditRipper ripper = new RedditRipper(new URL("https://www.reddit.com/user/ickybabie_")); + public void testSelfPostAuthorRip() throws IOException, URISyntaxException { + RedditRipper ripper = new RedditRipper(new URI("https://www.reddit.com/user/ickybabie_").toURL()); testRipper(ripper); } @@ -70,9 +71,9 @@ public class RedditRipperTest extends RippersTest { */ @Test @Tag("flaky") - public void testRedditGfyBadURL() throws IOException { + public void testRedditGfyBadURL() throws IOException, URISyntaxException { RedditRipper ripper = new RedditRipper( - new URL("https://www.reddit.com/r/bottesting/comments/7msmhi/bad_link/")); + new URI("https://www.reddit.com/r/bottesting/comments/7msmhi/bad_link/").toURL()); testRipper(ripper); } @@ -82,16 +83,16 @@ public class RedditRipperTest extends RippersTest { * @throws IOException */ @Test - public void testRedditGfycatRedirectURL() throws IOException { + public void testRedditGfycatRedirectURL() throws IOException, URISyntaxException { RedditRipper ripper = new RedditRipper( - new URL("https://www.reddit.com/r/NSFW_GIF/comments/ennwsa/gorgeous_tits/")); + new URI("https://www.reddit.com/r/NSFW_GIF/comments/ennwsa/gorgeous_tits/").toURL()); } @Test @Tag("flaky") - public void testRedditGallery() throws IOException{ + public void testRedditGallery() throws IOException, URISyntaxException { RedditRipper ripper = new RedditRipper( - new URL("https://www.reddit.com/gallery/hrrh23")); + new URI("https://www.reddit.com/gallery/hrrh23").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java index b6cec8e3..f89f9c40 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java @@ -6,7 +6,8 @@ import org.jsoup.nodes.Document; import org.junit.jupiter.api.*; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; public class RedgifsRipperTest extends RippersTest { @@ -15,8 +16,8 @@ public class RedgifsRipperTest extends RippersTest { */ @Test @Disabled("test or ripper broken") - public void testRedgifsGoodURL() throws IOException{ - RedgifsRipper ripper = new RedgifsRipper(new URL("https://www.redgifs.com/watch/talkativewarpeddragon-petite")); + public void testRedgifsGoodURL() throws IOException, URISyntaxException { + RedgifsRipper ripper = new RedgifsRipper(new URI("https://www.redgifs.com/watch/talkativewarpeddragon-petite").toURL()); testRipper(ripper); } @@ -25,8 +26,8 @@ public class RedgifsRipperTest extends RippersTest { */ @Test @Tag("flaky") - public void testRedgifsBadRL() throws IOException{ - RedgifsRipper ripper = new RedgifsRipper(new URL("https://www.gifdeliverynetwork.com/foolishelasticchimpanzee")); + public void testRedgifsBadRL() throws IOException, URISyntaxException { + RedgifsRipper ripper = new RedgifsRipper(new URI("https://www.gifdeliverynetwork.com/foolishelasticchimpanzee").toURL()); testRipper(ripper); } @@ -35,8 +36,8 @@ public class RedgifsRipperTest extends RippersTest { */ @Test @Tag("flaky") - public void testRedgifsProfile() throws IOException { - RedgifsRipper ripper = new RedgifsRipper(new URL("https://redgifs.com/users/margo_monty")); + public void testRedgifsProfile() throws IOException, URISyntaxException { + RedgifsRipper ripper = new RedgifsRipper(new URI("https://redgifs.com/users/margo_monty").toURL()); testRipper(ripper); } @@ -46,8 +47,8 @@ public class RedgifsRipperTest extends RippersTest { */ @Test @Disabled("test or ripper broken") - public void testRedgifsSearch() throws IOException { - RedgifsRipper ripper = new RedgifsRipper(new URL("https://redgifs.com/gifs/browse/little-caprice")); + public void testRedgifsSearch() throws IOException, URISyntaxException { + RedgifsRipper ripper = new RedgifsRipper(new URI("https://redgifs.com/gifs/browse/little-caprice").toURL()); Document doc = ripper.getFirstPage(); doc = ripper.getNextPage(doc); @@ -58,8 +59,8 @@ public class RedgifsRipperTest extends RippersTest { @Test @Tag("flaky") - public void testRedditRedgifs() throws IOException { - RedditRipper ripper = new RedditRipper(new URL("https://www.reddit.com/r/nsfwhardcore/comments/ouz5bw/me_cumming_on_his_face/")); + public void testRedditRedgifs() throws IOException, URISyntaxException { + RedditRipper ripper = new RedditRipper(new URI("https://www.reddit.com/r/nsfwhardcore/comments/ouz5bw/me_cumming_on_his_face/").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java index 89bd0fac..662a7eb7 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Rule34RipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.Rule34Ripper; @@ -9,14 +11,14 @@ import org.junit.jupiter.api.Test; public class Rule34RipperTest extends RippersTest { @Test - public void testShesFreakyRip() throws IOException { - Rule34Ripper ripper = new Rule34Ripper(new URL("https://rule34.xxx/index.php?page=post&s=list&tags=bimbo")); + public void testShesFreakyRip() throws IOException, URISyntaxException { + Rule34Ripper ripper = new Rule34Ripper(new URI("https://rule34.xxx/index.php?page=post&s=list&tags=bimbo").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("https://rule34.xxx/index.php?page=post&s=list&tags=bimbo"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://rule34.xxx/index.php?page=post&s=list&tags=bimbo").toURL(); Rule34Ripper ripper = new Rule34Ripper(url); Assertions.assertEquals("bimbo", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RulePornRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RulePornRipperTest.java index 03bbaa53..73f79a56 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RulePornRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RulePornRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.RulePornRipper; @@ -9,14 +11,14 @@ import org.junit.jupiter.api.Test; public class RulePornRipperTest extends RippersTest { @Test - public void testRip() throws IOException { - RulePornRipper ripper = new RulePornRipper(new URL("https://ruleporn.com/tosh/")); + public void testRip() throws IOException, URISyntaxException { + RulePornRipper ripper = new RulePornRipper(new URI("https://ruleporn.com/tosh/").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("https://ruleporn.com/tosh/"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://ruleporn.com/tosh/").toURL(); RulePornRipper ripper = new RulePornRipper(url); Assertions.assertEquals("tosh", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java index 88f59e2e..4efe9ba2 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SankakuComplexRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.SankakuComplexRipper; @@ -12,30 +14,30 @@ import org.junit.jupiter.api.Test; public class SankakuComplexRipperTest extends RippersTest { @Test @Disabled("https://github.com/RipMeApp/ripme/issues/257") - public void testSankakuChanRip() throws IOException { + public void testSankakuChanRip() throws IOException, URISyntaxException { SankakuComplexRipper ripper = new SankakuComplexRipper( - new URL("https://chan.sankakucomplex.com/?tags=cleavage")); + new URI("https://chan.sankakucomplex.com/?tags=cleavage").toURL()); testRipper(ripper); } @Test @Disabled("https://github.com/RipMeApp/ripme/issues/257") - public void testSankakuIdolRip() throws IOException { + public void testSankakuIdolRip() throws IOException, URISyntaxException { SankakuComplexRipper ripper = new SankakuComplexRipper( - new URL("https://idol.sankakucomplex.com/?tags=meme_%28me%21me%21me%21%29_%28cosplay%29")); + new URI("https://idol.sankakucomplex.com/?tags=meme_%28me%21me%21me%21%29_%28cosplay%29").toURL()); testRipper(ripper); } @Test - public void testgetGID() throws IOException { - URL url = new URL("https://idol.sankakucomplex.com/?tags=meme_%28me%21me%21me%21%29_%28cosplay%29"); + public void testgetGID() throws IOException, URISyntaxException { + URL url = new URI("https://idol.sankakucomplex.com/?tags=meme_%28me%21me%21me%21%29_%28cosplay%29").toURL(); SankakuComplexRipper ripper = new SankakuComplexRipper(url); Assertions.assertEquals("idol._meme_(me!me!me!)_(cosplay)", ripper.getGID(url)); } @Test - public void testgetSubDomain() throws IOException { - URL url = new URL("https://idol.sankakucomplex.com/?tags=meme_%28me%21me%21me%21%29_%28cosplay%29"); + public void testgetSubDomain() throws IOException, URISyntaxException { + URL url = new URI("https://idol.sankakucomplex.com/?tags=meme_%28me%21me%21me%21%29_%28cosplay%29").toURL(); SankakuComplexRipper ripper = new SankakuComplexRipper(url); Assertions.assertEquals("idol.", ripper.getSubDomain(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java index c7bf3d7d..f2b7461b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java @@ -5,19 +5,21 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.HashMap; import java.util.Map; public class ScrolllerRipperTest extends RippersTest { @Test - public void testScrolllerGID() throws IOException { + public void testScrolllerGID() throws IOException, URISyntaxException { Map testURLs = new HashMap<>(); - testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp"), "CatsStandingUp"); - testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=pictures"), "CatsStandingUp"); - testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?sort=top&filter=pictures"), "CatsStandingUp"); - testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=pictures&sort=top"), "CatsStandingUp"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp").toURL(), "CatsStandingUp"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures").toURL(), "CatsStandingUp"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=pictures").toURL(), "CatsStandingUp"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures&sort=top").toURL(), "CatsStandingUp"); for (URL url : testURLs.keySet()) { ScrolllerRipper ripper = new ScrolllerRipper(url); ripper.setup(); @@ -27,19 +29,19 @@ public class ScrolllerRipperTest extends RippersTest { } @Test - public void testScrolllerFilterRegex() throws IOException { + public void testScrolllerFilterRegex() throws IOException, URISyntaxException { Map testURLs = new HashMap<>(); - testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp"), "NOFILTER"); - testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=pictures"), "PICTURE"); - testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=videos"), "VIDEO"); - testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=albums"), "ALBUM"); - testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?sort=top&filter=pictures"), "PICTURE"); - testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?sort=top&filter=videos"), "VIDEO"); - testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?sort=top&filter=albums"), "ALBUM"); - testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=pictures&sort=top"), "PICTURE"); - testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=videos&sort=top"), "VIDEO"); - testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=albums&sort=top"), "ALBUM"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp").toURL(), "NOFILTER"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures").toURL(), "PICTURE"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=videos").toURL(), "VIDEO"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=albums").toURL(), "ALBUM"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=pictures").toURL(), "PICTURE"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=videos").toURL(), "VIDEO"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=albums").toURL(), "ALBUM"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures&sort=top").toURL(), "PICTURE"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=videos&sort=top").toURL(), "VIDEO"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=albums&sort=top").toURL(), "ALBUM"); for (URL url : testURLs.keySet()) { ScrolllerRipper ripper = new ScrolllerRipper(url); ripper.setup(); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java index eb3769a1..f389974b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ShesFreakyRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ShesFreakyRipper; @@ -12,15 +14,15 @@ import org.junit.jupiter.api.Test; public class ShesFreakyRipperTest extends RippersTest { @Test @Disabled("https://github.com/RipMeApp/ripme/issues/254") - public void testShesFreakyRip() throws IOException { + public void testShesFreakyRip() throws IOException, URISyntaxException { ShesFreakyRipper ripper = new ShesFreakyRipper( - new URL("http://www.shesfreaky.com/gallery/nicee-snow-bunny-579NbPjUcYa.html")); + new URI("http://www.shesfreaky.com/gallery/nicee-snow-bunny-579NbPjUcYa.html").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("http://www.shesfreaky.com/gallery/nicee-snow-bunny-579NbPjUcYa.html"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("http://www.shesfreaky.com/gallery/nicee-snow-bunny-579NbPjUcYa.html").toURL(); ShesFreakyRipper ripper = new ShesFreakyRipper(url); Assertions.assertEquals("nicee-snow-bunny-579NbPjUcYa", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TapasticRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TapasticRipperTest.java index 36a3a29e..4528482f 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TapasticRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TapasticRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.TapasticRipper; @@ -11,14 +13,14 @@ import org.junit.jupiter.api.Test; public class TapasticRipperTest extends RippersTest { @Test @Disabled("ripper broken") - public void testTapasticRip() throws IOException { - TapasticRipper ripper = new TapasticRipper(new URL("https://tapas.io/series/TPIAG")); + public void testTapasticRip() throws IOException, URISyntaxException { + TapasticRipper ripper = new TapasticRipper(new URI("https://tapas.io/series/TPIAG").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("https://tapas.io/series/TPIAG"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://tapas.io/series/TPIAG").toURL(); TapasticRipper ripper = new TapasticRipper(url); Assertions.assertEquals("series_ TPIAG", ripper.getGID(url)); } From d501578a8eccf8625166229c0e8281f31afe96b2 Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 13 Jun 2023 07:50:47 +0200 Subject: [PATCH 376/512] new URI instead of new URL in tests, 11. --- .../ripper/rippers/PhotobucketRipperTest.java | 24 ++++++++++--------- .../ripper/rippers/PichunterRipperTest.java | 11 +++++---- .../ripper/rippers/PorncomixRipperTest.java | 7 +++--- .../rippers/PorncomixinfoRipperTest.java | 7 +++--- .../tst/ripper/rippers/SinfestRipperTest.java | 10 ++++---- .../tst/ripper/rippers/SmuttyRipperTest.java | 10 ++++---- .../ripper/rippers/SpankBangRipperTest.java | 7 +++--- .../tst/ripper/rippers/StaRipperTest.java | 10 ++++---- .../ripper/rippers/StickyXXXRipperTest.java | 7 +++--- .../ripper/rippers/TeenplanetRipperTest.java | 10 ++++---- 10 files changed, 59 insertions(+), 44 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java index fb133d32..8581d038 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PhotobucketRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.PhotobucketRipper; @@ -13,9 +15,9 @@ public class PhotobucketRipperTest extends RippersTest { @Test @Disabled("https://github.com/RipMeApp/ripme/issues/229 : Disabled test (temporary) : BasicRippersTest#testPhotobucketRip (timing out)") - public void testPhotobucketRip() throws IOException { + public void testPhotobucketRip() throws IOException, URISyntaxException { PhotobucketRipper ripper = new PhotobucketRipper( - new URL("http://s844.photobucket.com/user/SpazzySpizzy/library/Album%20Covers?sort=3&page=1")); + new URI("http://s844.photobucket.com/user/SpazzySpizzy/library/Album%20Covers?sort=3&page=1").toURL()); testRipper(ripper); deleteSubdirs(ripper.getWorkingDir()); deleteDir(ripper.getWorkingDir()); @@ -23,12 +25,12 @@ public class PhotobucketRipperTest extends RippersTest { @Test @Disabled("new test, still disabled out because of the issue above, since this test also involves network IO.") - public void testGetNextPage() throws IOException { + public void testGetNextPage() throws IOException, URISyntaxException { // this album should have more than enough sub-albums and pages // to serve as a pretty good iteration test (barring server or // network errors) String baseURL = "http://s1255.photobucket.com/user/mimajki/library/Movie%20gifs?sort=6&page=1"; - URL url = new URL(baseURL); + URL url = new URI(baseURL).toURL(); PhotobucketRipper ripper = new PhotobucketRipper(url); org.jsoup.nodes.Document page = ripper.getFirstPage(); // NOTE: number of pages remaining includes the subalbums @@ -47,17 +49,17 @@ public class PhotobucketRipperTest extends RippersTest { } @Test - public void testGetGID() throws IOException { - URL url = new URL( - "http://s732.photobucket.com/user/doublesix66/library/Army%20Painter%20examples?sort=3&page=1"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI( + "http://s732.photobucket.com/user/doublesix66/library/Army%20Painter%20examples?sort=3&page=1").toURL(); PhotobucketRipper ripper = new PhotobucketRipper(url); Assertions.assertEquals("doublesix66", ripper.getGID(url)); - url = new URL( - "http://s732.photobucket.com/user/doublesix66/library/Army%20Painter%20examples/Painting%20examples?page=1&sort=3"); + url = new URI( + "http://s732.photobucket.com/user/doublesix66/library/Army%20Painter%20examples/Painting%20examples?page=1&sort=3").toURL(); Assertions.assertEquals("doublesix66", ripper.getGID(url)); - url = new URL("http://s844.photobucket.com/user/SpazzySpizzy/library/Album%20Covers"); + url = new URI("http://s844.photobucket.com/user/SpazzySpizzy/library/Album%20Covers").toURL(); Assertions.assertEquals("SpazzySpizzy", ripper.getGID(url)); - url = new URL("http://s844.photobucket.com/user/SpazzySpizzy/library"); + url = new URI("http://s844.photobucket.com/user/SpazzySpizzy/library").toURL(); Assertions.assertEquals("SpazzySpizzy", ripper.getGID(url)); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PichunterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PichunterRipperTest.java index 9ba9110b..fc79cb97 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PichunterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PichunterRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.PichunterRipper; @@ -11,18 +12,18 @@ import org.junit.jupiter.api.Test; public class PichunterRipperTest extends RippersTest { @Test @Tag("flaky") - public void testPichunterModelPageRip() throws IOException { + public void testPichunterModelPageRip() throws IOException, URISyntaxException { // A non-photoset - PichunterRipper ripper = new PichunterRipper(new URL("https://www.pichunter.com/models/Madison_Ivy")); + PichunterRipper ripper = new PichunterRipper(new URI("https://www.pichunter.com/models/Madison_Ivy").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testPichunterGalleryRip() throws IOException { + public void testPichunterGalleryRip() throws IOException, URISyntaxException { // a photo set PichunterRipper ripper = new PichunterRipper( - new URL("http://www.pichunter.com/gallery/3270642/Its_not_only_those_who")); + new URI("http://www.pichunter.com/gallery/3270642/Its_not_only_those_who").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixRipperTest.java index 7abe6e1e..ad9d9b83 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixRipperTest.java @@ -1,13 +1,14 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.PorncomixRipper; public class PorncomixRipperTest extends RippersTest { - public void testPorncomixAlbum() throws IOException { - PorncomixRipper ripper = new PorncomixRipper(new URL("http://www.porncomix.info/lust-unleashed-desire-to-submit/")); + public void testPorncomixAlbum() throws IOException, URISyntaxException { + PorncomixRipper ripper = new PorncomixRipper(new URI("http://www.porncomix.info/lust-unleashed-desire-to-submit/").toURL()); testRipper(ripper); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixinfoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixinfoRipperTest.java index 76841add..e8628955 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixinfoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixinfoRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.PorncomixinfoRipper; import org.junit.jupiter.api.Tag; @@ -10,8 +11,8 @@ import org.junit.jupiter.api.Test; public class PorncomixinfoRipperTest extends RippersTest { @Test @Tag("flaky") - public void testRip() throws IOException { - PorncomixinfoRipper ripper = new PorncomixinfoRipper(new URL("https://porncomixinfo.net/chapter/alx-come-to-naught-down-in-flames-up-in-smoke-tracy-scops/alx-come-to-naught-down-in-flames-up-in-smoke-tracy-scops/")); + public void testRip() throws IOException, URISyntaxException { + PorncomixinfoRipper ripper = new PorncomixinfoRipper(new URI("https://porncomixinfo.net/chapter/alx-come-to-naught-down-in-flames-up-in-smoke-tracy-scops/alx-come-to-naught-down-in-flames-up-in-smoke-tracy-scops/").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java index 905034c8..b7a8da53 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SinfestRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.SinfestRipper; @@ -11,14 +13,14 @@ import org.junit.jupiter.api.Test; public class SinfestRipperTest extends RippersTest { @Test @Tag("flaky") - public void testRip() throws IOException { - SinfestRipper ripper = new SinfestRipper(new URL("http://sinfest.net/view.php?date=2000-01-17")); + public void testRip() throws IOException, URISyntaxException { + SinfestRipper ripper = new SinfestRipper(new URI("http://sinfest.net/view.php?date=2000-01-17").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("http://sinfest.net/view.php?date=2000-01-17"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("http://sinfest.net/view.php?date=2000-01-17").toURL(); SinfestRipper ripper = new SinfestRipper(url); Assertions.assertEquals("2000-01-17", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java index 4085bb56..99c3f1aa 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.SmuttyRipper; @@ -11,14 +13,14 @@ import org.junit.jupiter.api.Test; public class SmuttyRipperTest extends RippersTest { @Test @Tag("flaky") - public void testRip() throws IOException { - SmuttyRipper ripper = new SmuttyRipper(new URL("https://smutty.com/user/QUIGON/")); + public void testRip() throws IOException, URISyntaxException { + SmuttyRipper ripper = new SmuttyRipper(new URI("https://smutty.com/user/QUIGON/").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("https://smutty.com/user/QUIGON/"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://smutty.com/user/QUIGON/").toURL(); SmuttyRipper ripper = new SmuttyRipper(url); Assertions.assertEquals("QUIGON", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SpankBangRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SpankBangRipperTest.java index c655bcf3..684d4689 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SpankBangRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SpankBangRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.SpankbangRipper; import org.junit.jupiter.api.Tag; @@ -10,8 +11,8 @@ import org.junit.jupiter.api.Test; public class SpankBangRipperTest extends RippersTest { @Test @Tag("flaky") - public void testSpankBangVideo() throws IOException { - SpankbangRipper ripper = new SpankbangRipper(new URL("https://spankbang.com/2a7fh/video/mdb901")); //most popular video of all time on site; should stay up + public void testSpankBangVideo() throws IOException, URISyntaxException { + SpankbangRipper ripper = new SpankbangRipper(new URI("https://spankbang.com/2a7fh/video/mdb901").toURL()); //most popular video of all time on site; should stay up testRipper(ripper); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java index 0ba05343..83da1175 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StaRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.StaRipper; @@ -12,15 +14,15 @@ import org.junit.jupiter.api.Test; public class StaRipperTest extends RippersTest { @Test @Disabled("Ripper broken, Nullpointer exception") - public void testRip() throws IOException { - StaRipper ripper = new StaRipper(new URL("https://sta.sh/01umpyuxi4js")); + public void testRip() throws IOException, URISyntaxException { + StaRipper ripper = new StaRipper(new URI("https://sta.sh/01umpyuxi4js").toURL()); testRipper(ripper); } @Test @Disabled - public void testGetGID() throws IOException { - URL url = new URL("https://sta.sh/01umpyuxi4js"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://sta.sh/01umpyuxi4js").toURL(); StaRipper ripper = new StaRipper(url); Assertions.assertEquals("01umpyuxi4js", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StickyXXXRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StickyXXXRipperTest.java index 5c530a01..57a07d90 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StickyXXXRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StickyXXXRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.video.StickyXXXRipper; // import com.rarchives.ripme.tst.ripper.rippers.RippersTest; @@ -9,10 +10,10 @@ import com.rarchives.ripme.utils.Utils; public class StickyXXXRipperTest extends RippersTest { - public void testStickyXXXVideo() throws IOException { + public void testStickyXXXVideo() throws IOException, URISyntaxException { // This test fails on the CI - possibly due to checking for a file before it's written - so we're skipping it if (Utils.getConfigBoolean("test.run_flaky_tests", false)) { - StickyXXXRipper ripper = new StickyXXXRipper(new URL("http://www.stickyxxx.com/a-very-intense-farewell/")); + StickyXXXRipper ripper = new StickyXXXRipper(new URI("http://www.stickyxxx.com/a-very-intense-farewell/").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java index 6e42b5db..2a69bae7 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TeenplanetRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.TeenplanetRipper; @@ -11,14 +13,14 @@ import org.junit.jupiter.api.Test; public class TeenplanetRipperTest extends RippersTest { @Test @Tag("flaky") - public void testTeenplanetRip() throws IOException { - TeenplanetRipper ripper = new TeenplanetRipper(new URL("http://teenplanet.org/galleries/the-perfect-side-of-me-6588.html")); + public void testTeenplanetRip() throws IOException, URISyntaxException { + TeenplanetRipper ripper = new TeenplanetRipper(new URI("http://teenplanet.org/galleries/the-perfect-side-of-me-6588.html").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("http://teenplanet.org/galleries/the-perfect-side-of-me-6588.html"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("http://teenplanet.org/galleries/the-perfect-side-of-me-6588.html").toURL(); TeenplanetRipper ripper = new TeenplanetRipper(url); Assertions.assertEquals("the-perfect-side-of-me-6588", ripper.getGID(url)); } From 44513b15a3d913ceeee666c2d0d8c4ee32c81ecc Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 13 Jun 2023 07:59:27 +0200 Subject: [PATCH 377/512] new URI instead of new URL in tests, 12. --- .../tst/ripper/rippers/ChanRipperTest.java | 6 +++--- .../ripper/rippers/PicstatioRipperTest.java | 13 ++++++++----- .../ripper/rippers/PornpicsRipperTest.java | 7 ++++--- .../ripper/rippers/SoundgasmRipperTest.java | 11 ++++++----- .../ripper/rippers/ThechiveRipperTest.java | 16 +++++++++------- .../rippers/TheyiffgalleryRipperTest.java | 10 ++++++---- .../tst/ripper/rippers/TsuminoRipperTest.java | 11 ++++++----- .../tst/ripper/rippers/TumblrRipperTest.java | 19 ++++++++++--------- .../tst/ripper/rippers/TwitterRipperTest.java | 11 ++++++----- .../rippers/TwodgalleriesRipperTest.java | 7 ++++--- .../tst/ripper/rippers/VidbleRipperTest.java | 10 ++++++---- .../tst/ripper/rippers/VideoRippersTest.java | 14 ++++++++------ .../ripper/rippers/ViewcomicRipperTest.java | 7 ++++--- .../tst/ripper/rippers/VkRipperTest.java | 19 ++++++++++--------- 14 files changed, 90 insertions(+), 71 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java index 078e366d..f64ada87 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java @@ -52,17 +52,17 @@ public class ChanRipperTest extends RippersTest { @Test public void testChanRipper() throws IOException, URISyntaxException { List contentURLs = new ArrayList<>(); - contentURLs.add(new URI(getRandomThreadDesuarchive()).toURL()); + contentURLs.add(getRandomThreadDesuarchive()); for (URL url : contentURLs) { ChanRipper ripper = new ChanRipper(url); testChanRipper(ripper); } } - public String getRandomThreadDesuarchive() throws URISyntaxException { + public URL getRandomThreadDesuarchive() throws URISyntaxException { try { Document doc = Http.url(new URI("https://desuarchive.org/wsg/").toURL()).get(); - return doc.select("div.post_data > a").first().attr("href"); + return new URI(doc.select("div.post_data > a").first().attr("href")).toURL(); } catch (IOException e) { e.printStackTrace(); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PicstatioRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PicstatioRipperTest.java index ec4e2383..04da17a8 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PicstatioRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PicstatioRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.PicstatioRipper; @@ -9,13 +11,14 @@ import org.junit.jupiter.api.Test; public class PicstatioRipperTest extends RippersTest { - public void testRip() throws IOException { - PicstatioRipper ripper = new PicstatioRipper(new URL("https://www.picstatio.com/aerial-view-wallpapers")); + public void testRip() throws IOException, URISyntaxException { + PicstatioRipper ripper = new PicstatioRipper(new URI("https://www.picstatio.com/aerial-view-wallpapers").toURL()); testRipper(ripper); } @Test - public void testGID() throws IOException { - PicstatioRipper ripper = new PicstatioRipper(new URL("https://www.picstatio.com/aerial-view-wallpapers")); - Assertions.assertEquals("aerial-view-wallpapers", ripper.getGID(new URL("https://www.picstatio.com/aerial-view-wallpapers"))); + public void testGID() throws IOException, URISyntaxException { + URL url = new URI("https://www.picstatio.com/aerial-view-wallpapers").toURL(); + PicstatioRipper ripper = new PicstatioRipper(url); + Assertions.assertEquals("aerial-view-wallpapers", ripper.getGID(url)); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornpicsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornpicsRipperTest.java index 1f79b254..4fda9aee 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornpicsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornpicsRipperTest.java @@ -1,13 +1,14 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.PornpicsRipper; public class PornpicsRipperTest extends RippersTest { - public void testRip() throws IOException { - PornpicsRipper ripper = new PornpicsRipper(new URL("https://www.pornpics.com/galleries/pornstar-dahlia-sky-takes-a-fat-cock-in-her-butthole-wearing-fishnet-stockings/")); + public void testRip() throws IOException, URISyntaxException { + PornpicsRipper ripper = new PornpicsRipper(new URI("https://www.pornpics.com/galleries/pornstar-dahlia-sky-takes-a-fat-cock-in-her-butthole-wearing-fishnet-stockings/").toURL()); testRipper(ripper); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java index 8a19fa12..877e9a04 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java @@ -6,21 +6,22 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; public class SoundgasmRipperTest extends RippersTest { @Test @Tag("flaky") - public void testSoundgasmURLs() throws IOException { - SoundgasmRipper ripper = new SoundgasmRipper(new URL("https://soundgasm.net/u/_Firefly_xoxo/Rambles-with-my-Lovense")); + public void testSoundgasmURLs() throws IOException, URISyntaxException { + SoundgasmRipper ripper = new SoundgasmRipper(new URI("https://soundgasm.net/u/_Firefly_xoxo/Rambles-with-my-Lovense").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testRedditSoundgasmURL() throws IOException { - RedditRipper ripper = new RedditRipper(new URL("https://www.reddit.com/r/gonewildaudio/comments/kn1bvj/f4m_mistress_controlled_my_lovense_while_i_tried/")); + public void testRedditSoundgasmURL() throws IOException, URISyntaxException { + RedditRipper ripper = new RedditRipper(new URI("https://www.reddit.com/r/gonewildaudio/comments/kn1bvj/f4m_mistress_controlled_my_lovense_while_i_tried/").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java index e7bdae85..c6884918 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java @@ -30,6 +30,8 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; /** @@ -45,17 +47,17 @@ public class ThechiveRipperTest extends RippersTest { */ @Test @Tag("flaky") - public void testTheChiveRip() throws IOException { - ThechiveRipper ripper = new ThechiveRipper(new URL( - "https://thechive.com/2019/03/16/beautiful-badasses-lookin-good-in-and-out-of-uniform-35-photos/")); + public void testTheChiveRip() throws IOException, URISyntaxException { + ThechiveRipper ripper = new ThechiveRipper(new URI( + "https://thechive.com/2019/03/16/beautiful-badasses-lookin-good-in-and-out-of-uniform-35-photos/").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testTheChiveGif() throws IOException { + public void testTheChiveGif() throws IOException, URISyntaxException { ThechiveRipper ripper = new ThechiveRipper( - new URL("https://thechive.com/2019/03/14/dont-tease-me-just-squeeze-me-20-gifs/")); + new URI("https://thechive.com/2019/03/14/dont-tease-me-just-squeeze-me-20-gifs/").toURL()); testRipper(ripper); } @@ -64,8 +66,8 @@ public class ThechiveRipperTest extends RippersTest { */ @Test @Tag("flaky") - public void testIDotThechive() throws IOException { - ThechiveRipper ripper = new ThechiveRipper(new URL("https://i.thechive.com/witcheva")); + public void testIDotThechive() throws IOException, URISyntaxException { + ThechiveRipper ripper = new ThechiveRipper(new URI("https://i.thechive.com/witcheva").toURL()); testRipper(ripper); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java index 3c9b6a4a..17ed7398 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TheyiffgalleryRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.TheyiffgalleryRipper; @@ -11,14 +13,14 @@ import org.junit.jupiter.api.Test; public class TheyiffgalleryRipperTest extends RippersTest { @Test @Tag("flaky") - public void testTheyiffgallery() throws IOException { - TheyiffgalleryRipper ripper = new TheyiffgalleryRipper(new URL("https://theyiffgallery.com/index?/category/4303")); + public void testTheyiffgallery() throws IOException, URISyntaxException { + TheyiffgalleryRipper ripper = new TheyiffgalleryRipper(new URI("https://theyiffgallery.com/index?/category/4303").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("https://theyiffgallery.com/index?/category/4303"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://theyiffgallery.com/index?/category/4303").toURL(); TheyiffgalleryRipper ripper = new TheyiffgalleryRipper(url); Assertions.assertEquals("4303", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TsuminoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TsuminoRipperTest.java index 38dee451..21818ae3 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TsuminoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TsuminoRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import java.util.List; import com.rarchives.ripme.ripper.rippers.TsuminoRipper; @@ -15,14 +16,14 @@ import org.junit.jupiter.api.Test; public class TsuminoRipperTest extends RippersTest { @Test @Disabled("Broken ripper") - public void testTsuminoRipper() throws IOException { - TsuminoRipper ripper = new TsuminoRipper(new URL("http://www.tsumino.com/Book/Info/43528/sore-wa-kurokute-suketeita-what-s-tight-and-black-and-sheer-all-over-")); + public void testTsuminoRipper() throws IOException, URISyntaxException { + TsuminoRipper ripper = new TsuminoRipper(new URI("http://www.tsumino.com/Book/Info/43528/sore-wa-kurokute-suketeita-what-s-tight-and-black-and-sheer-all-over-").toURL()); testRipper(ripper); } @Test @Disabled("Broken ripper") - public void testTagBlackList() throws IOException { - TsuminoRipper ripper = new TsuminoRipper(new URL("http://www.tsumino.com/Book/Info/43528/sore-wa-kurokute-suketeita-what-s-tight-and-black-and-sheer-all-over-")); + public void testTagBlackList() throws IOException, URISyntaxException { + TsuminoRipper ripper = new TsuminoRipper(new URI("http://www.tsumino.com/Book/Info/43528/sore-wa-kurokute-suketeita-what-s-tight-and-black-and-sheer-all-over-").toURL()); Document doc = ripper.getFirstPage(); List tagsOnPage = ripper.getTags(doc); String[] tags1 = {"test", "one", "Smell"}; diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java index 07aeb28d..e771e209 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java @@ -2,7 +2,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.TumblrRipper; @@ -12,30 +13,30 @@ import org.junit.jupiter.api.Test; public class TumblrRipperTest extends RippersTest { @Test @Disabled - public void testTumblrFullRip() throws IOException { - TumblrRipper ripper = new TumblrRipper(new URL("http://wrouinr.tumblr.com")); + public void testTumblrFullRip() throws IOException, URISyntaxException { + TumblrRipper ripper = new TumblrRipper(new URI("http://wrouinr.tumblr.com").toURL()); testRipper(ripper); } @Test @Disabled - public void testTumblrTagRip() throws IOException { - TumblrRipper ripper = new TumblrRipper(new URL("https://these-are-my-b-sides.tumblr.com/tagged/boobs")); + public void testTumblrTagRip() throws IOException, URISyntaxException { + TumblrRipper ripper = new TumblrRipper(new URI("https://these-are-my-b-sides.tumblr.com/tagged/boobs").toURL()); testRipper(ripper); } @Test @Disabled - public void testTumblrPostRip() throws IOException { - TumblrRipper ripper = new TumblrRipper(new URL("http://sadbaffoon.tumblr.com/post/132045920789/what-a-hoe")); + public void testTumblrPostRip() throws IOException, URISyntaxException { + TumblrRipper ripper = new TumblrRipper(new URI("http://sadbaffoon.tumblr.com/post/132045920789/what-a-hoe").toURL()); testRipper(ripper); } @Test @Disabled("Commented out because the test link is 404ing") - public void testEmbeddedImage() throws IOException { + public void testEmbeddedImage() throws IOException, URISyntaxException { TumblrRipper ripper = new TumblrRipper( - new URL("https://these-are-my-b-sides.tumblr.com/post/178225921524/this-was-fun")); + new URI("https://these-are-my-b-sides.tumblr.com/post/178225921524/this-was-fun").toURL()); testRipper(ripper); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java index 8e746c9e..de164767 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwitterRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.TwitterRipper; @@ -11,16 +12,16 @@ import org.junit.jupiter.api.Test; public class TwitterRipperTest extends RippersTest { @Test @Tag("flaky") - public void testTwitterUserRip() throws IOException { - TwitterRipper ripper = new TwitterRipper(new URL("https://twitter.com/danngamber01/media")); + public void testTwitterUserRip() throws IOException, URISyntaxException { + TwitterRipper ripper = new TwitterRipper(new URI("https://twitter.com/danngamber01/media").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testTwitterSearchRip() throws IOException { + public void testTwitterSearchRip() throws IOException, URISyntaxException { TwitterRipper ripper = new TwitterRipper( - new URL("https://twitter.com/search?f=tweets&q=from%3Aalinalixxx%20filter%3Aimages&src=typd")); + new URI("https://twitter.com/search?f=tweets&q=from%3Aalinalixxx%20filter%3Aimages&src=typd").toURL()); testRipper(ripper); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwodgalleriesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwodgalleriesRipperTest.java index 3671d506..1df43cff 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwodgalleriesRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TwodgalleriesRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.TwodgalleriesRipper; @@ -11,9 +12,9 @@ import org.junit.jupiter.api.Test; public class TwodgalleriesRipperTest extends RippersTest { @Test @Disabled("https://github.com/RipMeApp/ripme/issues/182") - public void testTwodgalleriesRip() throws IOException { + public void testTwodgalleriesRip() throws IOException, URISyntaxException { TwodgalleriesRipper ripper = new TwodgalleriesRipper( - new URL("http://www.2dgalleries.com/artist/regis-loisel-6477")); + new URI("http://www.2dgalleries.com/artist/regis-loisel-6477").toURL()); testRipper(ripper); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java index 58decfe6..7e3799f9 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.VidbleRipper; @@ -9,14 +11,14 @@ import org.junit.jupiter.api.Test; public class VidbleRipperTest extends RippersTest { @Test - public void testVidbleRip() throws IOException { - VidbleRipper ripper = new VidbleRipper(new URL("http://www.vidble.com/album/y1oyh3zd")); + public void testVidbleRip() throws IOException, URISyntaxException { + VidbleRipper ripper = new VidbleRipper(new URI("http://www.vidble.com/album/y1oyh3zd").toURL()); testRipper(ripper); } @Test - public void testGetGID() throws IOException { - URL url = new URL("http://www.vidble.com/album/y1oyh3zd"); + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("http://www.vidble.com/album/y1oyh3zd").toURL(); VidbleRipper ripper = new VidbleRipper(url); Assertions.assertEquals("y1oyh3zd", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java index fb162946..6ac08ca4 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VideoRippersTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -40,9 +42,9 @@ public class VideoRippersTest extends RippersTest { @Test @Disabled("Test disbaled. See https://github.com/RipMeApp/ripme/issues/574") - public void testTwitchVideoRipper() throws IOException { + public void testTwitchVideoRipper() throws IOException, URISyntaxException { List contentURLs = new ArrayList<>(); - contentURLs.add(new URL("https://clips.twitch.tv/FaithfulIncredulousPotTBCheesePull")); + contentURLs.add(new URI("https://clips.twitch.tv/FaithfulIncredulousPotTBCheesePull").toURL()); for (URL url : contentURLs) { // TwitchVideoRipper ripper = new TwitchVideoRipper(url); // videoTestHelper(ripper); @@ -51,18 +53,18 @@ public class VideoRippersTest extends RippersTest { @Test @Disabled("Test disabled see https://github.com/RipMeApp/ripme/issues/1095") - public void testPornhubRipper() throws IOException { + public void testPornhubRipper() throws IOException, URISyntaxException { List contentURLs = new ArrayList<>(); - contentURLs.add(new URL("https://www.pornhub.com/view_video.php?viewkey=ph5a329fa707269")); + contentURLs.add(new URI("https://www.pornhub.com/view_video.php?viewkey=ph5a329fa707269").toURL()); for (URL url : contentURLs) { PornhubRipper ripper = new PornhubRipper(url); videoTestHelper(ripper); } } - public void testYuvutuRipper() throws IOException { + public void testYuvutuRipper() throws IOException, URISyntaxException { List contentURLs = new ArrayList<>(); - contentURLs.add(new URL("http://www.yuvutu.com/video/828499/female-reader-armpit-job/")); + contentURLs.add(new URI("http://www.yuvutu.com/video/828499/female-reader-armpit-job/").toURL()); for (URL url : contentURLs) { YuvutuRipper ripper = new YuvutuRipper(url); videoTestHelper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ViewcomicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ViewcomicRipperTest.java index 063cc036..a315648d 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ViewcomicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ViewcomicRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.ViewcomicRipper; import org.junit.jupiter.api.Disabled; @@ -9,8 +10,8 @@ import org.junit.jupiter.api.Test; public class ViewcomicRipperTest extends RippersTest { @Test @Disabled("Ripper broken") - public void testViewcomicRipper() throws IOException { - ViewcomicRipper ripper = new ViewcomicRipper(new URL("https://view-comic.com/batman-no-mans-land-vol-1/")); + public void testViewcomicRipper() throws IOException, URISyntaxException { + ViewcomicRipper ripper = new ViewcomicRipper(new URI("https://view-comic.com/batman-no-mans-land-vol-1/").toURL()); testRipper(ripper); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java index 2580d85c..7bf7badf 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VkRipperTest.java @@ -1,7 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.VkRipper; import org.json.JSONObject; @@ -20,21 +21,21 @@ public class VkRipperTest extends RippersTest { // EXAMPLE: https://vk.com/album45506334_101886701 (a single album - custom) @Test @Tag("flaky") - public void testVkAlbumHttpRip() throws IOException { - VkRipper ripper = new VkRipper(new URL("https://vk.com/album45506334_0")); + public void testVkAlbumHttpRip() throws IOException, URISyntaxException { + VkRipper ripper = new VkRipper(new URI("https://vk.com/album45506334_0").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testVkPhotosRip() throws IOException { - VkRipper ripper = new VkRipper(new URL("https://vk.com/photos45506334")); + public void testVkPhotosRip() throws IOException, URISyntaxException { + VkRipper ripper = new VkRipper(new URI("https://vk.com/photos45506334").toURL()); testRipper(ripper); } @Test @Tag("flaky") - public void testFindJSONObjectContainingPhotoID() throws IOException { - VkRipper ripper = new VkRipper(new URL("http://vk.com/album45506334_0")); + public void testFindJSONObjectContainingPhotoID() throws IOException, URISyntaxException { + VkRipper ripper = new VkRipper(new URI("http://vk.com/album45506334_0").toURL()); String json = "{\"payload\":[0,[\"album-45984105_268691406\",18,14,[{\"id\":\"-45984105_457345201\",\"base\":\"https://sun9-37.userapi.com/\",\"tagged\":[],\"likes\":0,\"shares\":0,\"o_src\":\"https://sun9-65.userapi.com/c857520/v857520962/10e24c/DPxygc3XW5E.jpg\",\"o_\":[\"https://sun9-65.userapi.com/c857520/v857520962/10e24c/DPxygc3XW5E\",130,98],\"z_src\":\"https://sun9-41.userapi.com/c857520/v857520962/10e24a/EsDDQA36qKI.jpg\",\"z_\":[\"https://sun9-41.userapi.com/c857520/v857520962/10e24a/EsDDQA36qKI\",1280,960],\"w_src\":\"https://sun9-60.userapi.com/c857520/v857520962/10e24b/6ETsA15rAdU.jpg\",\"w_\":[\"https://sun9-60.userapi.com/c857520/v857520962/10e24b/6ETsA15rAdU\",1405,1054]}]]],\"langVersion\":\"4298\"}"; String responseJson = @@ -46,8 +47,8 @@ public class VkRipperTest extends RippersTest { } @Test - public void testGetBestSourceUrl() throws IOException { - VkRipper ripper = new VkRipper(new URL("http://vk.com/album45506334_0")); + public void testGetBestSourceUrl() throws IOException, URISyntaxException { + VkRipper ripper = new VkRipper(new URI("http://vk.com/album45506334_0").toURL()); String json = "{\"id\":\"-45984105_457345201\",\"base\":\"https://sun9-37.userapi.com/\",\"commcount\":0,\"date\":\"3 Dec at 1:14 am\",\"tagged\":[],\"attached_tags\":{\"max_tags_per_object\":5},\"o_src\":\"https://sun9-65.userapi.com/c857520/v857520962/10e24c/DPxygc3XW5E.jpg\",\"o_\":[\"https://sun9-65.userapi.com/c857520/v857520962/10e24c/DPxygc3XW5E\",130,98],\"y_src\":\"https://sun9-9.userapi.com/c857520/v857520962/10e249/dUDeuY10s0A.jpg\",\"y_\":[\"https://sun9-9.userapi.com/c857520/v857520962/10e249/dUDeuY10s0A\",807,605],\"z_src\":\"https://sun9-41.userapi.com/c857520/v857520962/10e24a/EsDDQA36qKI.jpg\",\"z_\":[\"https://sun9-41.userapi.com/c857520/v857520962/10e24a/EsDDQA36qKI\",1280,960]}"; Assertions.assertEquals("https://sun9-41.userapi.com/c857520/v857520962/10e24a/EsDDQA36qKI.jpg", From 59694711b02d4e3145f0f89447fef2a9d39f2a0e Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 13 Jun 2023 08:15:45 +0200 Subject: [PATCH 378/512] remove unused import. --- .../rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java index c6884918..e7d85d34 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java @@ -32,7 +32,6 @@ import org.junit.jupiter.api.Test; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.net.URL; /** * From 1a508ca4d8844a071d36cb30830f4dcc942711dc Mon Sep 17 00:00:00 2001 From: soloturn Date: Tue, 13 Jun 2023 14:04:18 +0200 Subject: [PATCH 379/512] correct vsco test --- .../com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java index aa963589..4fc14e8a 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java @@ -19,9 +19,8 @@ public class VscoRipperTest extends RippersTest { * @throws IOException */ @Test - @Tag("flaky") public void testSingleImageRip() throws IOException, URISyntaxException { - VscoRipper ripper = new VscoRipper(new URI("https://vsco.co/jonathangodoy/media/5d1aec76bb669a128035e98a").toURL()); + VscoRipper ripper = new VscoRipper(new URI("https://vsco.co/jolly-roger/media/597ce449846079297b3f7cf3").toURL()); testRipper(ripper); } From 430487b3030ea5790acb9f56f3559a92c1e4836e Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 15 Jun 2023 08:31:50 +0200 Subject: [PATCH 380/512] run tests with java-20 --- .github/workflows/gradle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 14e3b7a8..8c39b1f9 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - java: [11] + java: [20] include: # test newest java on one os only, upload from ubuntu java11 - os: ubuntu-latest java: 17 From 3ae8fd916aeae5596ff342599d4f5474766e0efe Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 15 Jun 2023 11:53:24 +0200 Subject: [PATCH 381/512] remove unused --- .../ripme/ripper/rippers/MyhentaicomicsRipper.java | 3 --- .../ripme/ripper/rippers/ScrolllerRipper.java | 2 +- src/test/java/com/rarchives/ripme/tst/proxyTest.java | 10 ++++++---- .../ripme/tst/ripper/rippers/ModelxRipperTest.java | 11 ++++++----- .../ripme/tst/ripper/rippers/VscoRipperTest.java | 1 - 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java index e206925f..cbe401cb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java @@ -13,8 +13,6 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; public class MyhentaicomicsRipper extends AbstractHTMLRipper { - private static boolean isTag; - public MyhentaicomicsRipper(URL url) throws IOException { super(url); } @@ -69,7 +67,6 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { Pattern pat = Pattern.compile("^https?://myhentaicomics.com/index.php/tag/([0-9]*)/?([a-zA-Z%0-9+?=:]*)?$"); Matcher mat = pat.matcher(url.toExternalForm()); if (mat.matches()) { - isTag = true; return true; } return false; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java index 7e0c1c46..614a575f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java @@ -120,7 +120,7 @@ public class ScrolllerRipper extends AbstractJSONRipper { try { String url = "https://api.scrolller.com/api/v2/graphql"; - URL obj = new URL(url); + URL obj = new URI(url).toURL(); HttpURLConnection conn = (HttpURLConnection) obj.openConnection(); conn.setReadTimeout(5000); conn.addRequestProperty("Accept-Language", "en-US,en;q=0.8"); diff --git a/src/test/java/com/rarchives/ripme/tst/proxyTest.java b/src/test/java/com/rarchives/ripme/tst/proxyTest.java index 72140853..7791290a 100644 --- a/src/test/java/com/rarchives/ripme/tst/proxyTest.java +++ b/src/test/java/com/rarchives/ripme/tst/proxyTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.utils.Proxy; import com.rarchives.ripme.utils.Utils; @@ -14,12 +16,12 @@ public class proxyTest { // This test will only run on machines where the user has added a entry for proxy.socks @Test - public void testSocksProxy() throws IOException { + public void testSocksProxy() throws IOException, URISyntaxException { // Unset proxy before testing System.setProperty("http.proxyHost", ""); System.setProperty("https.proxyHost", ""); System.setProperty("socksProxyHost", ""); - URL url = new URL("https://icanhazip.com"); + URL url = new URI("https://icanhazip.com").toURL(); String proxyConfig = Utils.getConfigString("proxy.socks", ""); if (!proxyConfig.equals("")) { String ip1 = Http.url(url).ignoreContentType().get().text(); @@ -34,12 +36,12 @@ public class proxyTest { // This test will only run on machines where the user has added a entry for proxy.http @Test - public void testHTTPProxy() throws IOException { + public void testHTTPProxy() throws IOException, URISyntaxException { // Unset proxy before testing System.setProperty("http.proxyHost", ""); System.setProperty("https.proxyHost", ""); System.setProperty("socksProxyHost", ""); - URL url = new URL("https://icanhazip.com"); + URL url = new URI("https://icanhazip.com").toURL(); String proxyConfig = Utils.getConfigString("proxy.http", ""); if (!proxyConfig.equals("")) { String ip1 = Http.url(url).ignoreContentType().get().text(); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelxRipperTest.java index 50fec37b..38f572a8 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelxRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelxRipperTest.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ModelxRipper; @@ -11,11 +13,10 @@ import org.junit.jupiter.api.Test; public class ModelxRipperTest extends RippersTest { @Test @Disabled("ModelxRipper domain has been changes. Commenting to avoid build failure.") - public void testModelxAlbum() throws IOException { - ModelxRipper ripper = new ModelxRipper(new URL( - "http://www.modelx.org/graphis-collection-2002-2016/ai-yuzuki-%e6%9f%9a%e6%9c%88%e3%81%82%e3%81%84-yuzuiro/")); - System.out.println(ripper.getGID(new URL( - "http://www.modelx.org/graphis-collection-2002-2016/ai-yuzuki-%e6%9f%9a%e6%9c%88%e3%81%82%e3%81%84-yuzuiro/"))); + public void testModelxAlbum() throws IOException, URISyntaxException { + URL url = new URI("http://www.modelx.org/graphis-collection-2002-2016/ai-yuzuki-%e6%9f%9a%e6%9c%88%e3%81%82%e3%81%84-yuzuiro/").toURL(); + ModelxRipper ripper = new ModelxRipper(url); + System.out.println(ripper.getGID(url)); testRipper(ripper); } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java index 4fc14e8a..c6ab6a79 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java @@ -3,7 +3,6 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.VscoRipper; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; From 7e6cdab4d7fb0878a11b206d6d737d82236434ff Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 15 Jun 2023 12:06:31 +0200 Subject: [PATCH 382/512] new URL is deprecated, use new URI, abstractripper --- .../com/rarchives/ripme/ripper/AbstractRipper.java | 3 ++- .../com/rarchives/ripme/ripper/RipperInterface.java | 3 ++- .../java/com/rarchives/ripme/ripper/VideoRipper.java | 3 ++- .../ripme/ripper/rippers/video/YuvutuRipper.java | 11 ++++------- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 4321d337..ff47174a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -8,6 +8,7 @@ import java.io.FileWriter; import java.io.IOException; import java.lang.reflect.Constructor; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; @@ -48,7 +49,7 @@ public abstract class AbstractRipper private boolean completed = true; - public abstract void rip() throws IOException; + public abstract void rip() throws IOException, URISyntaxException; public abstract String getHost(); public abstract String getGID(URL url) throws MalformedURLException; public boolean hasASAPRipping() { return false; } diff --git a/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java b/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java index 550209c0..63d48917 100644 --- a/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java +++ b/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; /** @@ -11,7 +12,7 @@ import java.net.URL; * (cheers!) */ interface RipperInterface { - void rip() throws IOException; + void rip() throws IOException, URISyntaxException; boolean canRip(URL url); URL sanitizeURL(URL url) throws MalformedURLException; void setWorkingDir(URL url) throws IOException; diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 54e624ce..cd4c95b3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -8,6 +8,7 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Path; import java.util.Map; @@ -22,7 +23,7 @@ public abstract class VideoRipper extends AbstractRipper { super(url); } - public abstract void rip() throws IOException; + public abstract void rip() throws IOException, URISyntaxException; public abstract String getHost(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java index 2891efb5..3fb55b6f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -34,11 +36,6 @@ public class YuvutuRipper extends VideoRipper { return m.matches(); } - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return url; - } - @Override public String getGID(URL url) throws MalformedURLException { Pattern p = Pattern.compile("^http://www\\.yuvutu\\.com/video/[0-9]+/(.*)$"); @@ -54,7 +51,7 @@ public class YuvutuRipper extends VideoRipper { } @Override - public void rip() throws IOException { + public void rip() throws IOException, URISyntaxException { LOGGER.info("Retrieving " + this.url); Document doc = Http.url(url).get(); Element iframe = doc.select("iframe").first(); @@ -74,7 +71,7 @@ public class YuvutuRipper extends VideoRipper { Matcher m = p.matcher(element.data()); if (m.find()){ String vidUrl = m.group(1); - addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url)); + addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); } } waitForThreads(); From 911f3d031130380de5b20f6a63b71fb8f0df4215 Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 15 Jun 2023 13:37:04 +0200 Subject: [PATCH 383/512] new URL is deprecated, additional exceptions --- .../java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 5 ++--- .../com/rarchives/ripme/ripper/rippers/MrCongRipper.java | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 83276aea..122c0ffe 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -3,8 +3,7 @@ package com.rarchives.ripme.ripper; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; +import java.net.*; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -104,7 +103,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { } @Override - public void rip() throws IOException { + public void rip() throws IOException, URISyntaxException { int index = 0; int textindex = 0; LOGGER.info("Retrieving " + this.url); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MrCongRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MrCongRipper.java index d0a7b571..642c6417 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MrCongRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MrCongRipper.java @@ -5,6 +5,7 @@ import com.rarchives.ripme.utils.Http; import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -214,7 +215,7 @@ public class MrCongRipper extends AbstractHTMLRipper { mcr.rip(); } - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { e.printStackTrace(); } } From 2e41ee0b356fd709739f3d9c5fe5e7d6ca4da7d6 Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 15 Jun 2023 14:24:07 +0200 Subject: [PATCH 384/512] sanitizeURL can throw URISyntaxException. --- .../java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 2 +- .../java/com/rarchives/ripme/ripper/AbstractRipper.java | 6 +++++- .../java/com/rarchives/ripme/ripper/RipperInterface.java | 2 +- .../com/rarchives/ripme/ripper/rippers/FlickrRipper.java | 6 ++++-- .../com/rarchives/ripme/ripper/rippers/LusciousRipper.java | 3 ++- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 122c0ffe..e5d09b8e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -73,7 +73,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { return url; } protected boolean hasDescriptionSupport() { diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index ff47174a..5772eb85 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -172,7 +172,11 @@ public abstract class AbstractRipper if (!canRip(url)) { throw new MalformedURLException("Unable to rip url: " + url); } - this.url = sanitizeURL(url); + try { + this.url = sanitizeURL(url); + } catch (URISyntaxException e) { + throw new MalformedURLException(e.getMessage()); + } } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java b/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java index 63d48917..cc994549 100644 --- a/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java +++ b/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java @@ -14,7 +14,7 @@ import java.net.URL; interface RipperInterface { void rip() throws IOException, URISyntaxException; boolean canRip(URL url); - URL sanitizeURL(URL url) throws MalformedURLException; + URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException; void setWorkingDir(URL url) throws IOException; String getHost(); String getGID(URL url) throws MalformedURLException; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java index 901a857c..f33c079e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.*; import java.util.regex.Matcher; @@ -62,7 +64,7 @@ public class FlickrRipper extends AbstractHTMLRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { String sUrl = url.toExternalForm(); // Strip out https sUrl = sUrl.replace("https://secure.flickr.com", "http://www.flickr.com"); @@ -73,7 +75,7 @@ public class FlickrRipper extends AbstractHTMLRipper { } sUrl += "pool"; } - return new URL(sUrl); + return new URI(sUrl).toURL(); } // FLickr is one of those sites what includes a api key in sites javascript // TODO let the user provide their own api key diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java index 277c5ff0..930eb38d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java @@ -9,6 +9,7 @@ import org.jsoup.select.Elements; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -84,7 +85,7 @@ public class LusciousRipper extends AbstractHTMLRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { // Sanitizes the url removing GET parameters and convert to legacy api url. // "https://legacy.luscious.net/albums/albumname" try { From 5aeaf7d2b04be4d0a28a2fbd53af76b0be872736 Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 15 Jun 2023 14:30:12 +0200 Subject: [PATCH 385/512] flickr test slow. --- .../rarchives/ripme/tst/ripper/rippers/FlickrRipperTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FlickrRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FlickrRipperTest.java index ea334447..22a507be 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FlickrRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FlickrRipperTest.java @@ -7,11 +7,12 @@ import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.FlickrRipper; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class FlickrRipperTest extends RippersTest { @Test - @Disabled("https://github.com/RipMeApp/ripme/issues/243") + @Tag("slow") public void testFlickrAlbum() throws IOException, URISyntaxException { FlickrRipper ripper = new FlickrRipper( new URI("https://www.flickr.com/photos/leavingallbehind/sets/72157621895942720/").toURL()); From 07178479e9595735d1c734570e296be4d2d69263 Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 15 Jun 2023 14:54:26 +0200 Subject: [PATCH 386/512] sanitizeURL can throw URISyntaxException, imgur. --- .../rarchives/ripme/ripper/AlbumRipper.java | 3 +- .../ripme/ripper/rippers/ImgurRipper.java | 102 +++++++++--------- .../com/rarchives/ripme/utils/RipUtils.java | 2 +- 3 files changed, 57 insertions(+), 50 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index c8efdb32..f952d010 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -7,6 +7,7 @@ import com.rarchives.ripme.utils.Utils; import java.io.File; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -33,7 +34,7 @@ public abstract class AlbumRipper extends AbstractRipper { } public abstract boolean canRip(URL url); - public abstract URL sanitizeURL(URL url) throws MalformedURLException; + public abstract URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException; public abstract void rip() throws IOException; public abstract String getHost(); public abstract String getGID(URL url) throws MalformedURLException; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index c854ae8c..a9aca156 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; @@ -72,7 +74,7 @@ public class ImgurRipper extends AlbumRipper { return true; } - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { String u = url.toExternalForm(); if (u.indexOf('#') >= 0) { u = u.substring(0, u.indexOf('#')); @@ -80,7 +82,7 @@ public class ImgurRipper extends AlbumRipper { u = u.replace("imgur.com/gallery/", "imgur.com/a/"); u = u.replace("https?://m\\.imgur\\.com", "http://imgur.com"); u = u.replace("https?://i\\.imgur\\.com", "http://imgur.com"); - return new URL(u); + return new URI(u).toURL(); } public String getAlbumTitle(URL url) throws MalformedURLException { @@ -145,40 +147,44 @@ public class ImgurRipper extends AlbumRipper { @Override public void rip() throws IOException { - switch (albumType) { - case ALBUM: - // Fall-through - case USER_ALBUM: - LOGGER.info("Album type is USER_ALBUM"); - // Don't call getAlbumTitle(this.url) with this - // as it seems to cause the album to be downloaded to a subdir. - ripAlbum(this.url); - break; - case SERIES_OF_IMAGES: - LOGGER.info("Album type is SERIES_OF_IMAGES"); - ripAlbum(this.url); - break; - case SINGLE_IMAGE: - LOGGER.info("Album type is SINGLE_IMAGE"); - ripSingleImage(this.url); - break; - case USER: - LOGGER.info("Album type is USER"); - ripUserAccount(url); - break; - case SUBREDDIT: - LOGGER.info("Album type is SUBREDDIT"); - ripSubreddit(url); - break; - case USER_IMAGES: - LOGGER.info("Album type is USER_IMAGES"); - ripUserImages(url); - break; + try { + switch (albumType) { + case ALBUM: + // Fall-through + case USER_ALBUM: + LOGGER.info("Album type is USER_ALBUM"); + // Don't call getAlbumTitle(this.url) with this + // as it seems to cause the album to be downloaded to a subdir. + ripAlbum(this.url); + break; + case SERIES_OF_IMAGES: + LOGGER.info("Album type is SERIES_OF_IMAGES"); + ripAlbum(this.url); + break; + case SINGLE_IMAGE: + LOGGER.info("Album type is SINGLE_IMAGE"); + ripSingleImage(this.url); + break; + case USER: + LOGGER.info("Album type is USER"); + ripUserAccount(url); + break; + case SUBREDDIT: + LOGGER.info("Album type is SUBREDDIT"); + ripSubreddit(url); + break; + case USER_IMAGES: + LOGGER.info("Album type is USER_IMAGES"); + ripUserImages(url); + break; + } + } catch (URISyntaxException e) { + throw new IOException("Failed ripping " + this.url, e); } waitForThreads(); } - private void ripSingleImage(URL url) throws IOException { + private void ripSingleImage(URL url) throws IOException, URISyntaxException { String strUrl = url.toExternalForm(); Document document = getDocument(strUrl); Matcher m = getEmbeddedJsonMatcher(document); @@ -188,11 +194,11 @@ public class ImgurRipper extends AlbumRipper { } } - private void ripAlbum(URL url) throws IOException { + private void ripAlbum(URL url) throws IOException, URISyntaxException { ripAlbum(url, ""); } - private void ripAlbum(URL url, String subdirectory) throws IOException { + private void ripAlbum(URL url, String subdirectory) throws IOException, URISyntaxException { int index; this.sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm()); index = 0; @@ -238,7 +244,7 @@ public class ImgurRipper extends AlbumRipper { continue; } String original = links.getString("original"); - ImgurImage theImage = new ImgurImage(new URL(original)); + ImgurImage theImage = new ImgurImage(new URI(original).toURL()); album.addImage(theImage); } catch (Exception e) { LOGGER.error("Got exception while fetching imgur ID " + imageId, e); @@ -248,7 +254,7 @@ public class ImgurRipper extends AlbumRipper { return album; } - public static ImgurAlbum getImgurAlbum(URL url) throws IOException { + public static ImgurAlbum getImgurAlbum(URL url) throws IOException, URISyntaxException { String strUrl = url.toExternalForm(); if (!strUrl.contains(",")) { strUrl += "/all"; @@ -262,7 +268,7 @@ public class ImgurRipper extends AlbumRipper { JSONObject json = new JSONObject(Jsoup.clean(doc.body().toString(), Safelist.none())); JSONArray jsonImages = json.getJSONObject("data").getJSONArray("images"); return createImgurAlbumFromJsonArray(url, jsonImages); - } catch (JSONException e) { + } catch (JSONException | URISyntaxException e) { LOGGER.debug("Error while parsing JSON at " + url + ", continuing", e); } @@ -296,7 +302,7 @@ public class ImgurRipper extends AlbumRipper { if (image.endsWith(".gif") && Utils.getConfigBoolean("prefer.mp4", false)) { image = image.replace(".gif", ".mp4"); } - ImgurImage imgurImage = new ImgurImage(new URL(image)); + ImgurImage imgurImage = new ImgurImage(new URI(image).toURL()); imgurAlbum.addImage(imgurImage); } return imgurAlbum; @@ -307,25 +313,25 @@ public class ImgurRipper extends AlbumRipper { return p.matcher(doc.body().html()); } - private static ImgurAlbum createImgurAlbumFromJsonArray(URL url, JSONArray jsonImages) throws MalformedURLException { + private static ImgurAlbum createImgurAlbumFromJsonArray(URL url, JSONArray jsonImages) throws MalformedURLException, URISyntaxException { ImgurAlbum imgurAlbum = new ImgurAlbum(url); int imagesLength = jsonImages.length(); for (int i = 0; i < imagesLength; i++) { JSONObject ob = jsonImages.getJSONObject(i); - imgurAlbum.addImage(new ImgurImage( new URL(ob.getString("link")))); + imgurAlbum.addImage(new ImgurImage( new URI(ob.getString("link")).toURL())); } return imgurAlbum; } - private static URL extractImageUrlFromJson(JSONObject json) throws MalformedURLException { + private static URL extractImageUrlFromJson(JSONObject json) throws MalformedURLException, URISyntaxException { String ext = json.getString("ext"); if (ext.equals(".gif") && Utils.getConfigBoolean("prefer.mp4", false)) { ext = ".mp4"; } - return new URL( + return new URI( "http://i.imgur.com/" + json.getString("hash") - + ext); + + ext).toURL(); } private static Document getDocument(String strUrl) throws IOException { @@ -352,7 +358,7 @@ public class ImgurRipper extends AlbumRipper { * @param url * URL to imgur user account (http://username.imgur.com) */ - private void ripUserAccount(URL url) throws IOException { + private void ripUserAccount(URL url) throws IOException, URISyntaxException { LOGGER.info("Retrieving " + url); sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm()); Document doc = Http.url(url).get(); @@ -363,7 +369,7 @@ public class ImgurRipper extends AlbumRipper { continue; } String albumID = album.attr("href").substring(album.attr("href").lastIndexOf('/') + 1); - URL albumURL = new URL("http:" + album.attr("href") + "/noscript"); + URL albumURL = new URI("http:" + album.attr("href") + "/noscript").toURL(); try { ripAlbum(albumURL, albumID); Thread.sleep(SLEEP_BETWEEN_ALBUMS * 1000L); @@ -398,7 +404,7 @@ public class ImgurRipper extends AlbumRipper { if (Utils.getConfigBoolean("download.save_order", true)) { prefix = String.format("%03d_", imagesFound); } - addURLToDownload(new URL(imageUrl), prefix); + addURLToDownload(new URI(imageUrl).toURL(), prefix); } if (imagesFound >= imagesTotal) { break; @@ -411,7 +417,7 @@ public class ImgurRipper extends AlbumRipper { } } - private void ripSubreddit(URL url) throws IOException { + private void ripSubreddit(URL url) throws IOException, URISyntaxException { int page = 0; while (true) { stopCheck(); @@ -431,7 +437,7 @@ public class ImgurRipper extends AlbumRipper { if (image.contains("b.")) { image = image.replace("b.", "."); } - URL imageURL = new URL(image); + URL imageURL = new URI(image).toURL(); addURLToDownload(imageURL); } if (imgs.isEmpty()) { diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index f0aa0d18..14aa6dce 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -42,7 +42,7 @@ public class RipUtils { logger.debug("Got imgur image: " + imgurImage.url); result.add(imgurImage.url); } - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { logger.error("[!] Exception while loading album " + url, e); } return result; From 029b03c74dac73bbab1078f6fec57ca483786298 Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 15 Jun 2023 15:07:09 +0200 Subject: [PATCH 387/512] new URL is deprecated, use new URI, setGID, imgur --- .../rarchives/ripme/ripper/AbstractRipper.java | 8 ++++++-- .../com/rarchives/ripme/ripper/AlbumRipper.java | 2 +- .../rarchives/ripme/ripper/RipperInterface.java | 2 +- .../ripme/ripper/rippers/ImgurRipper.java | 15 ++++++++++----- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 5772eb85..1a25af18 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -51,7 +51,7 @@ public abstract class AbstractRipper public abstract void rip() throws IOException, URISyntaxException; public abstract String getHost(); - public abstract String getGID(URL url) throws MalformedURLException; + public abstract String getGID(URL url) throws MalformedURLException, URISyntaxException; public boolean hasASAPRipping() { return false; } // Everytime addUrlToDownload skips a already downloaded url this increases by 1 public int alreadyDownloadedUrls = 0; @@ -551,7 +551,11 @@ public abstract class AbstractRipper * If any of those damned URLs gets malformed. */ public String getAlbumTitle(URL url) throws MalformedURLException { - return getHost() + "_" + getGID(url); + try { + return getHost() + "_" + getGID(url); + } catch (URISyntaxException e) { + throw new MalformedURLException(e.getMessage()); + } } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index f952d010..96c352f5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -37,7 +37,7 @@ public abstract class AlbumRipper extends AbstractRipper { public abstract URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException; public abstract void rip() throws IOException; public abstract String getHost(); - public abstract String getGID(URL url) throws MalformedURLException; + public abstract String getGID(URL url) throws MalformedURLException, URISyntaxException; protected boolean allowDuplicates() { return false; diff --git a/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java b/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java index cc994549..67572898 100644 --- a/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java +++ b/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java @@ -17,5 +17,5 @@ interface RipperInterface { URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException; void setWorkingDir(URL url) throws IOException; String getHost(); - String getGID(URL url) throws MalformedURLException; + String getGID(URL url) throws MalformedURLException, URISyntaxException; } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index a9aca156..b383b97e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -86,7 +86,12 @@ public class ImgurRipper extends AlbumRipper { } public String getAlbumTitle(URL url) throws MalformedURLException { - String gid = getGID(url); + String gid = null; + try { + gid = getGID(url); + } catch (URISyntaxException e) { + throw new MalformedURLException(e.getMessage()); + } if (this.albumType == ALBUM_TYPE.ALBUM) { try { // Attempt to use album title as GID @@ -459,7 +464,7 @@ public class ImgurRipper extends AlbumRipper { } @Override - public String getGID(URL url) throws MalformedURLException { + public String getGID(URL url) throws MalformedURLException, URISyntaxException { Pattern p; Matcher m; @@ -469,7 +474,7 @@ public class ImgurRipper extends AlbumRipper { // Imgur album or gallery albumType = ALBUM_TYPE.ALBUM; String gid = m.group(m.groupCount()); - this.url = new URL("http://imgur.com/a/" + gid); + this.url = new URI("http://imgur.com/a/" + gid).toURL(); return gid; } p = Pattern.compile("^https?://(www\\.|m\\.)?imgur\\.com/(a|gallery|t)/[a-zA-Z0-9]*/([a-zA-Z0-9]{5,}).*$"); @@ -478,7 +483,7 @@ public class ImgurRipper extends AlbumRipper { // Imgur album or gallery albumType = ALBUM_TYPE.ALBUM; String gid = m.group(m.groupCount()); - this.url = new URL("http://imgur.com/a/" + gid); + this.url = new URI("http://imgur.com/a/" + gid).toURL(); return gid; } p = Pattern.compile("^https?://([a-zA-Z0-9\\-]{3,})\\.imgur\\.com/?$"); @@ -526,7 +531,7 @@ public class ImgurRipper extends AlbumRipper { albumType = ALBUM_TYPE.ALBUM; String subreddit = m.group(m.groupCount() - 1); String gid = m.group(m.groupCount()); - this.url = new URL("http://imgur.com/r/" + subreddit + "/" + gid); + this.url = new URI("http://imgur.com/r/" + subreddit + "/" + gid).toURL(); return "r_" + subreddit + "_" + gid; } p = Pattern.compile("^https?://(i\\.|www\\.|m\\.)?imgur\\.com/([a-zA-Z0-9]{5,})$"); From 6ffd6f34d0b8973e7a899bf3f91f9eabd6c75d2a Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 15 Jun 2023 15:17:21 +0200 Subject: [PATCH 388/512] new URL is deprecated, use new URI, reddit. --- .../ripme/ripper/rippers/RedditRipper.java | 54 ++++++++++--------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 12a11b6e..dcfa14e7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -3,6 +3,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.io.OutputStream; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; @@ -56,19 +58,19 @@ public class RedditRipper extends AlbumRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { String u = url.toExternalForm(); // Strip '/u/' from URL u = u.replaceAll("reddit\\.com/u/", "reddit.com/user/"); - return new URL(u); + return new URI(u).toURL(); } - private URL getJsonURL(URL url) throws MalformedURLException { + private URL getJsonURL(URL url) throws MalformedURLException, URISyntaxException { // Convert gallery to post link and append ".json" Pattern p = Pattern.compile("^https?://[a-zA-Z0-9.]{0,4}reddit\\.com/gallery/([a-zA-Z0-9]+).*$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { - return new URL("https://reddit.com/" +m.group(m.groupCount())+ ".json"); + return new URI("https://reddit.com/" +m.group(m.groupCount())+ ".json").toURL(); } // Append ".json" to URL in appropriate location. @@ -76,28 +78,32 @@ public class RedditRipper extends AlbumRipper { if (url.getQuery() != null) { result += "?" + url.getQuery(); } - return new URL(result); + return new URI(result).toURL(); } @Override public void rip() throws IOException { - URL jsonURL = getJsonURL(this.url); - while (true) { - if (shouldAddURL()) { - sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_COMPLETE_HISTORY, "Already seen the last " + alreadyDownloadedUrls + " images ending rip"); - break; - } - jsonURL = getAndParseAndReturnNext(jsonURL); - if (jsonURL == null || isThisATest() || isStopped()) { - break; + try { + URL jsonURL = getJsonURL(this.url); + while (true) { + if (shouldAddURL()) { + sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_COMPLETE_HISTORY, "Already seen the last " + alreadyDownloadedUrls + " images ending rip"); + break; + } + jsonURL = getAndParseAndReturnNext(jsonURL); + if (jsonURL == null || isThisATest() || isStopped()) { + break; + } } + } catch (URISyntaxException e) { + new IOException(e.getMessage()); } waitForThreads(); } - private URL getAndParseAndReturnNext(URL url) throws IOException { + private URL getAndParseAndReturnNext(URL url) throws IOException, URISyntaxException { JSONArray jsonArray = getJsonArrayFromURL(url), children; JSONObject json, data; URL nextURL = null; @@ -118,7 +124,7 @@ public class RedditRipper extends AlbumRipper { if (children.getJSONObject(j).getString("kind").equals("t3") && children.getJSONObject(j).getJSONObject("data").getBoolean("is_self") ) { - URL selfPostURL = new URL(children.getJSONObject(j).getJSONObject("data").getString("url")); + URL selfPostURL = new URI(children.getJSONObject(j).getJSONObject("data").getString("url")).toURL(); System.out.println(selfPostURL.toExternalForm()); saveText(getJsonArrayFromURL(getJsonURL(selfPostURL))); } @@ -134,7 +140,7 @@ public class RedditRipper extends AlbumRipper { else { nextURLString = nextURLString.concat("?after=" + data.getString("after")); } - nextURL = new URL(nextURLString); + nextURL = new URI(nextURLString).toURL(); } } @@ -378,8 +384,8 @@ public class RedditRipper extends AlbumRipper { baseURL = doc.select("MPD > Period > AdaptationSet > Representation[height=" + height + "]").select("BaseURL").text(); } } - return new URL(vidURL + "/" + baseURL); - } catch (IOException e) { + return new URI(vidURL + "/" + baseURL).toURL(); + } catch (IOException | URISyntaxException e) { e.printStackTrace(); } return null; @@ -389,8 +395,8 @@ public class RedditRipper extends AlbumRipper { private void handleURL(String theUrl, String id, String title) { URL originalURL; try { - originalURL = new URL(theUrl); - } catch (MalformedURLException e) { + originalURL = new URI(theUrl).toURL(); + } catch (MalformedURLException | URISyntaxException e) { return; } String subdirectory = ""; @@ -455,12 +461,12 @@ public class RedditRipper extends AlbumRipper { try { URL mediaURL; if (!media.getJSONObject("s").isNull("gif")) { - mediaURL = new URL(media.getJSONObject("s").getString("gif").replaceAll("&", "&")); + mediaURL = new URI(media.getJSONObject("s").getString("gif").replaceAll("&", "&")).toURL(); } else { - mediaURL = new URL(media.getJSONObject("s").getString("u").replaceAll("&", "&")); + mediaURL = new URI(media.getJSONObject("s").getString("u").replaceAll("&", "&")).toURL(); } addURLToDownload(mediaURL, prefix, subdirectory); - } catch (MalformedURLException | JSONException e) { + } catch (MalformedURLException | JSONException | URISyntaxException e) { LOGGER.error("[!] Unable to parse gallery JSON:\ngallery_data:\n" + data +"\nmedia_metadata:\n" + metadata); } } From ab17fa9822c15d0d90ec2daa9b368c29f1e30998 Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 15 Jun 2023 16:21:33 +0200 Subject: [PATCH 389/512] new URL is deprecated, use new URI, AbstractHTML. --- .../java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 2 +- .../rarchives/ripme/ripper/rippers/ComicextraRipper.java | 3 ++- .../com/rarchives/ripme/ripper/rippers/EHentaiRipper.java | 6 ++++-- .../com/rarchives/ripme/ripper/rippers/ListalRipper.java | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index e5d09b8e..6a944d8e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -51,7 +51,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { return cachedFirstPage; } - public Document getNextPage(Document doc) throws IOException { + public Document getNextPage(Document doc) throws IOException, URISyntaxException { return null; } protected abstract List getURLsFromPage(Document page); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ComicextraRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ComicextraRipper.java index a2e79bc0..3abf749a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ComicextraRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ComicextraRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -96,7 +97,7 @@ public class ComicextraRipper extends AbstractHTMLRipper { } @Override - public Document getNextPage(Document doc) throws IOException { + public Document getNextPage(Document doc) throws IOException, URISyntaxException { if (urlType == UrlType.COMIC) { ++chapterIndex; imageIndex = 0; // Resetting the imagesIndex so that images prefix within each chapter starts from '001_'. diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java index 97373e5e..33165c93 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java @@ -13,6 +13,8 @@ import org.jsoup.select.Elements; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; @@ -154,7 +156,7 @@ public class EHentaiRipper extends AbstractHTMLRipper { } @Override - public Document getNextPage(Document doc) throws IOException { + public Document getNextPage(Document doc) throws IOException, URISyntaxException { // Check if we've stopped if (isStopped()) { throw new IOException("Ripping interrupted"); @@ -174,7 +176,7 @@ public class EHentaiRipper extends AbstractHTMLRipper { // Sleep before loading next page sleep(PAGE_SLEEP_TIME); // Load next page - Document nextPage = getPageWithRetries(new URL(nextURL)); + Document nextPage = getPageWithRetries(new URI(nextURL).toURL()); this.lastURL = nextURL; return nextPage; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java index 235da1c7..408310a7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java @@ -96,7 +96,7 @@ public class ListalRipper extends AbstractHTMLRipper { } @Override - public Document getNextPage(Document page) throws IOException { + public Document getNextPage(Document page) throws IOException, URISyntaxException { Document nextPage = super.getNextPage(page); switch (urlType) { case LIST: From 96c9907ec4d7472f934b31ae678200f6f791cbe9 Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 15 Jun 2023 18:20:46 +0200 Subject: [PATCH 390/512] dos2unix to have lf line endings. --- .github/ISSUE_TEMPLATE.md | 30 +- .github/PULL_REQUEST_TEMPLATE.md | 54 +- .../ripme/ripper/rippers/ArtstnRipper.java | 116 ++-- .../ripper/rippers/ComicextraRipper.java | 348 +++++------ .../ripme/ripper/rippers/CyberdropRipper.java | 108 ++-- .../ripper/rippers/FitnakedgirlsRipper.java | 130 ++-- .../ripme/ripper/rippers/ScrolllerRipper.java | 584 +++++++++--------- .../ripme/ripper/rippers/VscoRipper.java | 446 ++++++------- .../ripme/ripper/rippers/XlecxRipper.java | 72 +-- .../rippers/video/TwitchVideoRipper.java | 158 ++--- .../java/com/rarchives/ripme/utils/Proxy.java | 198 +++--- .../resources/LabelsBundle_pl_PL.properties | 116 ++-- .../resources/LabelsBundle_zh_CN.properties | 148 ++--- .../com/rarchives/ripme/tst/proxyTest.java | 114 ++-- .../ripper/rippers/ComicextraRipperTest.java | 56 +- .../ripper/rippers/CyberdropRipperTest.java | 108 ++-- .../tst/ripper/rippers/FolioRipperTest.java | 60 +- .../tst/ripper/rippers/GfycatRipperTest.java | 106 ++-- .../tst/ripper/rippers/ListalRipperTest.java | 84 +-- .../ripper/rippers/ScrolllerRipperTest.java | 110 ++-- .../tst/ripper/rippers/VscoRipperTest.java | 104 ++-- .../tst/ripper/rippers/XlecxRipperTest.java | 38 +- utils/style.sh | 27 - utils/stylefix.sh | 17 - 24 files changed, 1644 insertions(+), 1688 deletions(-) delete mode 100644 utils/style.sh delete mode 100644 utils/stylefix.sh diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 06283ebf..0c095b28 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,15 +1,15 @@ -* Ripme version: -* Java version: -* Operating system: - -* Exact URL you were trying to rip when the problem occurred: -* Please include any additional information about how to reproduce the problem: - -## Expected Behavior - -Detail the expected behavior here. - -## Actual Behavior - -Detail the actual (incorrect) behavior here. You can post log snippets or attach log files to your issue report. +* Ripme version: +* Java version: +* Operating system: + +* Exact URL you were trying to rip when the problem occurred: +* Please include any additional information about how to reproduce the problem: + +## Expected Behavior + +Detail the expected behavior here. + +## Actual Behavior + +Detail the actual (incorrect) behavior here. You can post log snippets or attach log files to your issue report. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 8810800c..56d0dd29 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,27 +1,27 @@ -# Category - -This change is exactly one of the following (please change `[ ]` to `[x]`) to indicate which: -* [ ] a bug fix (Fix #...) -* [ ] a new Ripper -* [ ] a refactoring -* [ ] a style change/fix -* [ ] a new feature - - -# Description - -Please add details about your change here. - - -# Testing - -Required verification: -* [ ] I've verified that there are no regressions in `mvn test` (there are no new failures or errors). -* [ ] I've verified that this change works as intended. - * [ ] Downloads all relevant content. - * [ ] Downloads content from multiple pages (as necessary or appropriate). - * [ ] Saves content at reasonable file names (e.g. page titles or content IDs) to help easily browse downloaded content. -* [ ] I've verified that this change did not break existing functionality (especially in the Ripper I modified). - -Optional but recommended: -* [ ] I've added a unit test to cover my change. +# Category + +This change is exactly one of the following (please change `[ ]` to `[x]`) to indicate which: +* [ ] a bug fix (Fix #...) +* [ ] a new Ripper +* [ ] a refactoring +* [ ] a style change/fix +* [ ] a new feature + + +# Description + +Please add details about your change here. + + +# Testing + +Required verification: +* [ ] I've verified that there are no regressions in `mvn test` (there are no new failures or errors). +* [ ] I've verified that this change works as intended. + * [ ] Downloads all relevant content. + * [ ] Downloads content from multiple pages (as necessary or appropriate). + * [ ] Saves content at reasonable file names (e.g. page titles or content IDs) to help easily browse downloaded content. +* [ ] I've verified that this change did not break existing functionality (especially in the Ripper I modified). + +Optional but recommended: +* [ ] I've added a unit test to cover my change. diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ArtstnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtstnRipper.java index 82b6e97c..7505c078 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ArtstnRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtstnRipper.java @@ -1,58 +1,58 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; - -import org.jsoup.Connection.Response; - -import com.rarchives.ripme.utils.Http; - -/* - * Ripper for ArtStation's short URL domain. - * Example URL: https://artstn.co/p/JlE15Z - */ - -public class ArtstnRipper extends ArtStationRipper { - public URL artStationUrl = null; - - public ArtstnRipper(URL url) throws IOException { - super(url); - } - - @Override - public boolean canRip(URL url) { - return url.getHost().endsWith("artstn.co"); - } - - @Override - public String getGID(URL url) throws MalformedURLException { - if (artStationUrl == null) { - // Run only once. - try { - artStationUrl = getFinalUrl(url); - if (artStationUrl == null) { - throw new IOException("Null url received."); - } - } catch (IOException e) { - LOGGER.error("Couldnt resolve URL.", e); - } - - } - return super.getGID(artStationUrl); - } - - public URL getFinalUrl(URL url) throws IOException { - if (url.getHost().endsWith("artstation.com")) { - return url; - } - - LOGGER.info("Checking url: " + url); - Response response = Http.url(url).connection().followRedirects(false).execute(); - if (response.statusCode() / 100 == 3 && response.hasHeader("location")) { - return getFinalUrl(new URL(response.header("location"))); - } else { - return null; - } - } -} +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; + +import org.jsoup.Connection.Response; + +import com.rarchives.ripme.utils.Http; + +/* + * Ripper for ArtStation's short URL domain. + * Example URL: https://artstn.co/p/JlE15Z + */ + +public class ArtstnRipper extends ArtStationRipper { + public URL artStationUrl = null; + + public ArtstnRipper(URL url) throws IOException { + super(url); + } + + @Override + public boolean canRip(URL url) { + return url.getHost().endsWith("artstn.co"); + } + + @Override + public String getGID(URL url) throws MalformedURLException { + if (artStationUrl == null) { + // Run only once. + try { + artStationUrl = getFinalUrl(url); + if (artStationUrl == null) { + throw new IOException("Null url received."); + } + } catch (IOException e) { + LOGGER.error("Couldnt resolve URL.", e); + } + + } + return super.getGID(artStationUrl); + } + + public URL getFinalUrl(URL url) throws IOException { + if (url.getHost().endsWith("artstation.com")) { + return url; + } + + LOGGER.info("Checking url: " + url); + Response response = Http.url(url).connection().followRedirects(false).execute(); + if (response.statusCode() / 100 == 3 && response.hasHeader("location")) { + return getFinalUrl(new URL(response.header("location"))); + } else { + return null; + } + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ComicextraRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ComicextraRipper.java index 3abf749a..e794e072 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ComicextraRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ComicextraRipper.java @@ -1,174 +1,174 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; - -/** - * @author Tushar - * - */ -public class ComicextraRipper extends AbstractHTMLRipper { - - private static final String FILE_NAME = "page"; - - private Pattern p1 = - Pattern.compile("https:\\/\\/www.comicextra.com\\/comic\\/([A-Za-z0-9_-]+)"); - private Pattern p2 = Pattern.compile( - "https:\\/\\/www.comicextra.com\\/([A-Za-z0-9_-]+)\\/([A-Za-z0-9_-]+)(?:\\/full)?"); - private UrlType urlType = UrlType.UNKNOWN; - private List chaptersList = null; - private int chapterIndex = -1; // index for the chaptersList, useful in getting the next page. - private int imageIndex = 0; // image index for each chapter images. - - public ComicextraRipper(URL url) throws IOException { - super(url); - } - - @Override - protected String getDomain() { - return "comicextra.com"; - } - - @Override - public String getHost() { - return "comicextra"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Matcher m1 = p1.matcher(url.toExternalForm()); - if (m1.matches()) { - // URL is of comic( https://www.comicextra.com/comic/the-punisher-frank-castle-max). - urlType = UrlType.COMIC; - return m1.group(1); - } - - Matcher m2 = p2.matcher(url.toExternalForm()); - if (m2.matches()) { - // URL is of chapter( https://www.comicextra.com/the-punisher-frank-castle-max/chapter-75). - urlType = UrlType.CHAPTER; - return m2.group(1); - } - - throw new MalformedURLException( - "Expected comicextra.com url of type: https://www.comicextra.com/comic/some-comic-name\n" - + " or https://www.comicextra.com/some-comic-name/chapter-001 got " + url - + " instead"); - } - - @Override - protected Document getFirstPage() throws IOException { - Document doc = null; - - switch (urlType) { - case COMIC: - // For COMIC type url we extract the urls of each chapters and store them in chapters. - chaptersList = new ArrayList<>(); - Document comicPage = Http.url(url).get(); - Elements elements = comicPage.select("div.episode-list a"); - for (Element e : elements) { - chaptersList.add(getCompleteChapterUrl(e.attr("abs:href"))); - } - - // Set the first chapter from the chapterList as the doc. - chapterIndex = 0; - doc = Http.url(chaptersList.get(chapterIndex)).get(); - break; - case CHAPTER: - doc = Http.url(url).get(); - break; - case UNKNOWN: - default: - throw new IOException("Unknown url type encountered."); - } - - return doc; - } - - @Override - public Document getNextPage(Document doc) throws IOException, URISyntaxException { - if (urlType == UrlType.COMIC) { - ++chapterIndex; - imageIndex = 0; // Resetting the imagesIndex so that images prefix within each chapter starts from '001_'. - if (chapterIndex < chaptersList.size()) { - return Http.url(chaptersList.get(chapterIndex)).get(); - } - } - - return super.getNextPage(doc); - } - - @Override - protected List getURLsFromPage(Document page) { - List urls = new ArrayList<>(); - - if (urlType == UrlType.COMIC || urlType == UrlType.CHAPTER) { - Elements images = page.select("img.chapter_img"); - for (Element img : images) { - urls.add(img.attr("src")); - } - } - - return urls; - } - - @Override - protected void downloadURL(URL url, int index) { - String subdirectory = getSubDirectoryName(); - String prefix = getPrefix(++imageIndex); - - addURLToDownload(url, subdirectory, null, null, prefix, FILE_NAME, null, Boolean.TRUE); - } - - /* - * This function appends /full at the end of the chapters url to get all the images for the - * chapter in the same Document. - */ - private String getCompleteChapterUrl(String chapterUrl) { - if (!chapterUrl.endsWith("/full")) { - chapterUrl = chapterUrl + "/full"; - } - return chapterUrl; - } - - /* - * This functions returns sub folder name for the current chapter. - */ - private String getSubDirectoryName() { - String subDirectory = ""; - - if (urlType == UrlType.COMIC) { - Matcher m = p2.matcher(chaptersList.get(chapterIndex)); - if (m.matches()) { - subDirectory = m.group(2); - } - } - - if (urlType == UrlType.CHAPTER) { - Matcher m = p2.matcher(url.toExternalForm()); - if (m.matches()) { - subDirectory = m.group(2); - } - } - - return subDirectory; - } - - /* - * Enum to classify different types of urls. - */ - private enum UrlType { - COMIC, CHAPTER, UNKNOWN - } -} +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +/** + * @author Tushar + * + */ +public class ComicextraRipper extends AbstractHTMLRipper { + + private static final String FILE_NAME = "page"; + + private Pattern p1 = + Pattern.compile("https:\\/\\/www.comicextra.com\\/comic\\/([A-Za-z0-9_-]+)"); + private Pattern p2 = Pattern.compile( + "https:\\/\\/www.comicextra.com\\/([A-Za-z0-9_-]+)\\/([A-Za-z0-9_-]+)(?:\\/full)?"); + private UrlType urlType = UrlType.UNKNOWN; + private List chaptersList = null; + private int chapterIndex = -1; // index for the chaptersList, useful in getting the next page. + private int imageIndex = 0; // image index for each chapter images. + + public ComicextraRipper(URL url) throws IOException { + super(url); + } + + @Override + protected String getDomain() { + return "comicextra.com"; + } + + @Override + public String getHost() { + return "comicextra"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Matcher m1 = p1.matcher(url.toExternalForm()); + if (m1.matches()) { + // URL is of comic( https://www.comicextra.com/comic/the-punisher-frank-castle-max). + urlType = UrlType.COMIC; + return m1.group(1); + } + + Matcher m2 = p2.matcher(url.toExternalForm()); + if (m2.matches()) { + // URL is of chapter( https://www.comicextra.com/the-punisher-frank-castle-max/chapter-75). + urlType = UrlType.CHAPTER; + return m2.group(1); + } + + throw new MalformedURLException( + "Expected comicextra.com url of type: https://www.comicextra.com/comic/some-comic-name\n" + + " or https://www.comicextra.com/some-comic-name/chapter-001 got " + url + + " instead"); + } + + @Override + protected Document getFirstPage() throws IOException { + Document doc = null; + + switch (urlType) { + case COMIC: + // For COMIC type url we extract the urls of each chapters and store them in chapters. + chaptersList = new ArrayList<>(); + Document comicPage = Http.url(url).get(); + Elements elements = comicPage.select("div.episode-list a"); + for (Element e : elements) { + chaptersList.add(getCompleteChapterUrl(e.attr("abs:href"))); + } + + // Set the first chapter from the chapterList as the doc. + chapterIndex = 0; + doc = Http.url(chaptersList.get(chapterIndex)).get(); + break; + case CHAPTER: + doc = Http.url(url).get(); + break; + case UNKNOWN: + default: + throw new IOException("Unknown url type encountered."); + } + + return doc; + } + + @Override + public Document getNextPage(Document doc) throws IOException, URISyntaxException { + if (urlType == UrlType.COMIC) { + ++chapterIndex; + imageIndex = 0; // Resetting the imagesIndex so that images prefix within each chapter starts from '001_'. + if (chapterIndex < chaptersList.size()) { + return Http.url(chaptersList.get(chapterIndex)).get(); + } + } + + return super.getNextPage(doc); + } + + @Override + protected List getURLsFromPage(Document page) { + List urls = new ArrayList<>(); + + if (urlType == UrlType.COMIC || urlType == UrlType.CHAPTER) { + Elements images = page.select("img.chapter_img"); + for (Element img : images) { + urls.add(img.attr("src")); + } + } + + return urls; + } + + @Override + protected void downloadURL(URL url, int index) { + String subdirectory = getSubDirectoryName(); + String prefix = getPrefix(++imageIndex); + + addURLToDownload(url, subdirectory, null, null, prefix, FILE_NAME, null, Boolean.TRUE); + } + + /* + * This function appends /full at the end of the chapters url to get all the images for the + * chapter in the same Document. + */ + private String getCompleteChapterUrl(String chapterUrl) { + if (!chapterUrl.endsWith("/full")) { + chapterUrl = chapterUrl + "/full"; + } + return chapterUrl; + } + + /* + * This functions returns sub folder name for the current chapter. + */ + private String getSubDirectoryName() { + String subDirectory = ""; + + if (urlType == UrlType.COMIC) { + Matcher m = p2.matcher(chaptersList.get(chapterIndex)); + if (m.matches()) { + subDirectory = m.group(2); + } + } + + if (urlType == UrlType.CHAPTER) { + Matcher m = p2.matcher(url.toExternalForm()); + if (m.matches()) { + subDirectory = m.group(2); + } + } + + return subDirectory; + } + + /* + * Enum to classify different types of urls. + */ + private enum UrlType { + COMIC, CHAPTER, UNKNOWN + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java index 0e0220f9..81a39823 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java @@ -1,55 +1,55 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.*; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -public class CyberdropRipper extends AbstractHTMLRipper { - - public CyberdropRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "cyberdrop"; - } - - @Override - public String getDomain() { - return "cyberdrop.me"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://cyberdrop\\.me/a/([a-zA-Z0-9]+).*?$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - throw new MalformedURLException("Expected cyberdrop.me URL format: " + - "https://cyberdrop.me/a/xxxxxxxx - got " + url + "instead"); - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } - - @Override - protected List getURLsFromPage(Document page) { - ArrayList urls = new ArrayList<>(); - for (Element element: page.getElementsByClass("image")) { - urls.add(element.attr("href")); - } - return urls; - } +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.*; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +public class CyberdropRipper extends AbstractHTMLRipper { + + public CyberdropRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "cyberdrop"; + } + + @Override + public String getDomain() { + return "cyberdrop.me"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^https?://cyberdrop\\.me/a/([a-zA-Z0-9]+).*?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected cyberdrop.me URL format: " + + "https://cyberdrop.me/a/xxxxxxxx - got " + url + "instead"); + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + + @Override + protected List getURLsFromPage(Document page) { + ArrayList urls = new ArrayList<>(); + for (Element element: page.getElementsByClass("image")) { + urls.add(element.attr("href")); + } + return urls; + } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FitnakedgirlsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FitnakedgirlsRipper.java index 3c0a68f0..51d5f15f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FitnakedgirlsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FitnakedgirlsRipper.java @@ -1,66 +1,66 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; - -public class FitnakedgirlsRipper extends AbstractHTMLRipper { - - public FitnakedgirlsRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "fitnakedgirls"; - } - - @Override - public String getDomain() { - return "fitnakedgirls.com"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p; - Matcher m; - - p = Pattern.compile("^.*fitnakedgirls\\.com/gallery/(.+)$"); - m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - - throw new MalformedURLException( - "Expected fitnakedgirls.com gallery format: " + "fitnakedgirls.com/gallery/####" + " Got: " + url); - } - - @Override - public List getURLsFromPage(Document doc) { - List imageURLs = new ArrayList<>(); - - Elements imgs = doc.select("div[class*=wp-tiles-tile-bg] > img"); - for (Element img : imgs) { - String imgSrc = img.attr("src"); - imageURLs.add(imgSrc); - } - - return imageURLs; - } - - @Override - public void downloadURL(URL url, int index) { - // Send referrer when downloading images - addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), null); - } +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; + +public class FitnakedgirlsRipper extends AbstractHTMLRipper { + + public FitnakedgirlsRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "fitnakedgirls"; + } + + @Override + public String getDomain() { + return "fitnakedgirls.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p; + Matcher m; + + p = Pattern.compile("^.*fitnakedgirls\\.com/gallery/(.+)$"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + + throw new MalformedURLException( + "Expected fitnakedgirls.com gallery format: " + "fitnakedgirls.com/gallery/####" + " Got: " + url); + } + + @Override + public List getURLsFromPage(Document doc) { + List imageURLs = new ArrayList<>(); + + Elements imgs = doc.select("div[class*=wp-tiles-tile-bg] > img"); + for (Element img : imgs) { + String imgSrc = img.attr("src"); + imageURLs.add(imgSrc); + } + + return imageURLs; + } + + @Override + public void downloadURL(URL url, int index) { + // Send referrer when downloading images + addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), null); + } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java index 614a575f..2df6ab2c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java @@ -1,293 +1,293 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.net.*; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.java_websocket.client.WebSocketClient; - -import org.apache.http.NameValuePair; -import org.apache.http.client.utils.URLEncodedUtils; -import org.java_websocket.handshake.ServerHandshake; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -import com.rarchives.ripme.ripper.AbstractJSONRipper; - -public class ScrolllerRipper extends AbstractJSONRipper { - - public ScrolllerRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "scrolller"; - } - @Override - public String getDomain() { - return "scrolller.com"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - // Typical URL is: https://scrolller.com/r/subreddit - // Parameters like "filter" and "sort" can be passed (ex: https://scrolller.com/r/subreddit?filter=xxx&sort=yyyy) - Pattern p = Pattern.compile("^https?://scrolller\\.com/r/([a-zA-Z0-9]+).*?$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - throw new MalformedURLException("Expected scrolller.com URL format: " + - "scrolller.com/r/subreddit OR scroller.com/r/subreddit?filter= - got " + url + "instead"); - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } - - - private JSONObject prepareQuery(String iterator, String gid, String sortByString) throws IOException, URISyntaxException { - - String QUERY_NOSORT = "query SubredditQuery( $url: String! $filter: SubredditPostFilter $iterator: String ) { getSubreddit(url: $url) { children( limit: 50 iterator: $iterator filter: $filter ) { iterator items { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } } } }"; - String QUERY_SORT = "subscription SubredditSubscription( $url: String! $sortBy: SubredditSortBy $timespan: SubredditTimespan $iterator: String $limit: Int $filter: SubredditPostFilter ) { fetchSubreddit( url: $url sortBy: $sortBy timespan: $timespan iterator: $iterator limit: $limit filter: $filter ) { __typename ... on Subreddit { __typename url title secondaryTitle description createdAt isNsfw subscribers isComplete itemCount videoCount pictureCount albumCount isFollowing } ... on SubredditPost { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } ... on Iterator { iterator } ... on Error { message } } }"; - - String filterString = convertFilterString(getParameter(this.url,"filter")); - - JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid)).put("sortBy", sortByString.toUpperCase()); - JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", sortByString.equals("") ? QUERY_NOSORT : QUERY_SORT); - - if (iterator != null) { - // Iterator is not present on the first page - variablesObject.put("iterator", iterator); - } - if (!filterString.equals("NOFILTER")) { - variablesObject.put("filter", filterString); - } - - return sortByString.equals("") ? getPosts(finalQueryObject) : getPostsSorted(finalQueryObject); - - } - - - public String convertFilterString(String filterParameter) { - // Converts the ?filter= parameter of the URL to one that can be used in the GraphQL query - // I could basically remove the last "s" and call toUpperCase instead of this switch statement but this looks easier to read. - switch (filterParameter.toLowerCase()) { - case "pictures": - return "PICTURE"; - case "videos": - return "VIDEO"; - case "albums": - return "ALBUM"; - case "": - return "NOFILTER"; - default: - LOGGER.error(String.format("Invalid filter %s using no filter",filterParameter)); - return ""; - } - } - - public String getParameter(URL url, String parameter) throws MalformedURLException { - // Gets passed parameters from the URL - String toReplace = String.format("https://scrolller.com/r/%s?",getGID(url)); - List args= URLEncodedUtils.parse(url.toExternalForm(), Charset.defaultCharset()); - for (NameValuePair arg:args) { - // First parameter contains part of the url so we have to remove it - // Ex: for the url https://scrolller.com/r/CatsStandingUp?filter=xxxx&sort=yyyy - // 1) arg.getName() => https://scrolller.com/r/CatsStandingUp?filter - // 2) arg.getName() => sort - - if (arg.getName().replace(toReplace,"").toLowerCase().equals((parameter))) { - return arg.getValue(); - } - } - return ""; - } - - private JSONObject getPosts(JSONObject data) { - // The actual GraphQL query call - - try { - String url = "https://api.scrolller.com/api/v2/graphql"; - - URL obj = new URI(url).toURL(); - HttpURLConnection conn = (HttpURLConnection) obj.openConnection(); - conn.setReadTimeout(5000); - conn.addRequestProperty("Accept-Language", "en-US,en;q=0.8"); - conn.addRequestProperty("User-Agent", "Mozilla"); - conn.addRequestProperty("Referer", "scrolller.com"); - - conn.setDoOutput(true); - - OutputStreamWriter w = new OutputStreamWriter(conn.getOutputStream(), "UTF-8"); - - w.write(data.toString()); - w.close(); - - BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream())); - String inputLine; - StringBuffer jsonString = new StringBuffer(); - - while ((inputLine = in.readLine()) != null) { - jsonString.append(inputLine); - } - - in.close(); - conn.disconnect(); - - return new JSONObject(jsonString.toString()); - - } catch (Exception e) { - e.printStackTrace(); - } - - return new JSONObject("{}"); - } - - private JSONObject getPostsSorted(JSONObject data) throws MalformedURLException { - - // The actual GraphQL query call (if sort parameter is present) - try { - - ArrayList postsJsonStrings = new ArrayList<>(); - - WebSocketClient wsc = new WebSocketClient(new URI("wss://api.scrolller.com/api/v2/graphql")) { - @Override - public void onOpen(ServerHandshake serverHandshake) { - // As soon as the WebSocket connects send our query - this.send(data.toString()); - } - - @Override - public void onMessage(String s) { - postsJsonStrings.add(s); - if (new JSONObject(s).getJSONObject("data").getJSONObject("fetchSubreddit").has("iterator")) { - this.close(); - } - } - - @Override - public void onClose(int i, String s, boolean b) { - } - - @Override - public void onError(Exception e) { - LOGGER.error(String.format("WebSocket error, server reported %s", e.getMessage())); - } - }; - wsc.connect(); - - while (!wsc.isClosed()) { - // Posts list is not over until the connection closes. - } - - JSONObject finalObject = new JSONObject(); - JSONArray posts = new JSONArray(); - - // Iterator is the last object in the post list, let's duplicate it in his own object for clarity. - finalObject.put("iterator", new JSONObject(postsJsonStrings.get(postsJsonStrings.size()-1))); - - for (String postString : postsJsonStrings) { - posts.put(new JSONObject(postString)); - } - finalObject.put("posts", posts); - - if (finalObject.getJSONArray("posts").length() == 1 && !finalObject.getJSONArray("posts").getJSONObject(0).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { - // Only iterator, no posts. - return null; - } - - return finalObject; - - - } catch (URISyntaxException ue) { - // Nothing to catch, it's an hardcoded URI. - } - - return null; - } - - - @Override - protected List getURLsFromJSON(JSONObject json) throws JSONException { - - boolean sortRequested = json.has("posts"); - - int bestArea = 0; - String bestUrl = ""; - List list = new ArrayList<>(); - - JSONArray itemsList = sortRequested ? json.getJSONArray("posts") : json.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").getJSONArray("items"); - - for (Object item : itemsList) { - - if (sortRequested && !((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { - continue; - } - - JSONArray sourcesTMP = sortRequested ? ((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").getJSONArray("mediaSources") : ((JSONObject) item).getJSONArray("mediaSources"); - for (Object sourceTMP : sourcesTMP) - { - int widthTMP = ((JSONObject) sourceTMP).getInt("width"); - int heightTMP = ((JSONObject) sourceTMP).getInt("height"); - int areaTMP = widthTMP * heightTMP; - - if (areaTMP > bestArea) { - bestArea = widthTMP; - bestUrl = ((JSONObject) sourceTMP).getString("url"); - } - } - list.add(bestUrl); - bestUrl = ""; - bestArea = 0; - } - - return list; - } - - @Override - protected JSONObject getFirstPage() throws IOException { - try { - return prepareQuery(null, this.getGID(url), getParameter(url,"sort")); - } catch (URISyntaxException e) { - LOGGER.error(String.format("Error obtaining first page: %s", e.getMessage())); - return null; - } - } - - @Override - public JSONObject getNextPage(JSONObject source) throws IOException { - // Every call the the API contains an "iterator" string that we need to pass to the API to get the next page - // Checking if iterator is null is not working for some reason, hence why the weird "iterator.toString().equals("null")" - - Object iterator = null; - if (source.has("iterator")) { - // Sort requested, custom JSON. - iterator = source.getJSONObject("iterator").getJSONObject("data").getJSONObject("fetchSubreddit").get("iterator"); - } else { - iterator = source.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").get("iterator"); - } - - if (!iterator.toString().equals("null")) { - // Need to change page. - try { - return prepareQuery(iterator.toString(), this.getGID(url), getParameter(url,"sort")); - } catch (URISyntaxException e) { - LOGGER.error(String.format("Error changing page: %s", e.getMessage())); - return null; - } - } else { - return null; - } - } +package com.rarchives.ripme.ripper.rippers; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.net.*; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.java_websocket.client.WebSocketClient; + +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; +import org.java_websocket.handshake.ServerHandshake; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import com.rarchives.ripme.ripper.AbstractJSONRipper; + +public class ScrolllerRipper extends AbstractJSONRipper { + + public ScrolllerRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "scrolller"; + } + @Override + public String getDomain() { + return "scrolller.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + // Typical URL is: https://scrolller.com/r/subreddit + // Parameters like "filter" and "sort" can be passed (ex: https://scrolller.com/r/subreddit?filter=xxx&sort=yyyy) + Pattern p = Pattern.compile("^https?://scrolller\\.com/r/([a-zA-Z0-9]+).*?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected scrolller.com URL format: " + + "scrolller.com/r/subreddit OR scroller.com/r/subreddit?filter= - got " + url + "instead"); + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + + + private JSONObject prepareQuery(String iterator, String gid, String sortByString) throws IOException, URISyntaxException { + + String QUERY_NOSORT = "query SubredditQuery( $url: String! $filter: SubredditPostFilter $iterator: String ) { getSubreddit(url: $url) { children( limit: 50 iterator: $iterator filter: $filter ) { iterator items { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } } } }"; + String QUERY_SORT = "subscription SubredditSubscription( $url: String! $sortBy: SubredditSortBy $timespan: SubredditTimespan $iterator: String $limit: Int $filter: SubredditPostFilter ) { fetchSubreddit( url: $url sortBy: $sortBy timespan: $timespan iterator: $iterator limit: $limit filter: $filter ) { __typename ... on Subreddit { __typename url title secondaryTitle description createdAt isNsfw subscribers isComplete itemCount videoCount pictureCount albumCount isFollowing } ... on SubredditPost { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } ... on Iterator { iterator } ... on Error { message } } }"; + + String filterString = convertFilterString(getParameter(this.url,"filter")); + + JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid)).put("sortBy", sortByString.toUpperCase()); + JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", sortByString.equals("") ? QUERY_NOSORT : QUERY_SORT); + + if (iterator != null) { + // Iterator is not present on the first page + variablesObject.put("iterator", iterator); + } + if (!filterString.equals("NOFILTER")) { + variablesObject.put("filter", filterString); + } + + return sortByString.equals("") ? getPosts(finalQueryObject) : getPostsSorted(finalQueryObject); + + } + + + public String convertFilterString(String filterParameter) { + // Converts the ?filter= parameter of the URL to one that can be used in the GraphQL query + // I could basically remove the last "s" and call toUpperCase instead of this switch statement but this looks easier to read. + switch (filterParameter.toLowerCase()) { + case "pictures": + return "PICTURE"; + case "videos": + return "VIDEO"; + case "albums": + return "ALBUM"; + case "": + return "NOFILTER"; + default: + LOGGER.error(String.format("Invalid filter %s using no filter",filterParameter)); + return ""; + } + } + + public String getParameter(URL url, String parameter) throws MalformedURLException { + // Gets passed parameters from the URL + String toReplace = String.format("https://scrolller.com/r/%s?",getGID(url)); + List args= URLEncodedUtils.parse(url.toExternalForm(), Charset.defaultCharset()); + for (NameValuePair arg:args) { + // First parameter contains part of the url so we have to remove it + // Ex: for the url https://scrolller.com/r/CatsStandingUp?filter=xxxx&sort=yyyy + // 1) arg.getName() => https://scrolller.com/r/CatsStandingUp?filter + // 2) arg.getName() => sort + + if (arg.getName().replace(toReplace,"").toLowerCase().equals((parameter))) { + return arg.getValue(); + } + } + return ""; + } + + private JSONObject getPosts(JSONObject data) { + // The actual GraphQL query call + + try { + String url = "https://api.scrolller.com/api/v2/graphql"; + + URL obj = new URI(url).toURL(); + HttpURLConnection conn = (HttpURLConnection) obj.openConnection(); + conn.setReadTimeout(5000); + conn.addRequestProperty("Accept-Language", "en-US,en;q=0.8"); + conn.addRequestProperty("User-Agent", "Mozilla"); + conn.addRequestProperty("Referer", "scrolller.com"); + + conn.setDoOutput(true); + + OutputStreamWriter w = new OutputStreamWriter(conn.getOutputStream(), "UTF-8"); + + w.write(data.toString()); + w.close(); + + BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream())); + String inputLine; + StringBuffer jsonString = new StringBuffer(); + + while ((inputLine = in.readLine()) != null) { + jsonString.append(inputLine); + } + + in.close(); + conn.disconnect(); + + return new JSONObject(jsonString.toString()); + + } catch (Exception e) { + e.printStackTrace(); + } + + return new JSONObject("{}"); + } + + private JSONObject getPostsSorted(JSONObject data) throws MalformedURLException { + + // The actual GraphQL query call (if sort parameter is present) + try { + + ArrayList postsJsonStrings = new ArrayList<>(); + + WebSocketClient wsc = new WebSocketClient(new URI("wss://api.scrolller.com/api/v2/graphql")) { + @Override + public void onOpen(ServerHandshake serverHandshake) { + // As soon as the WebSocket connects send our query + this.send(data.toString()); + } + + @Override + public void onMessage(String s) { + postsJsonStrings.add(s); + if (new JSONObject(s).getJSONObject("data").getJSONObject("fetchSubreddit").has("iterator")) { + this.close(); + } + } + + @Override + public void onClose(int i, String s, boolean b) { + } + + @Override + public void onError(Exception e) { + LOGGER.error(String.format("WebSocket error, server reported %s", e.getMessage())); + } + }; + wsc.connect(); + + while (!wsc.isClosed()) { + // Posts list is not over until the connection closes. + } + + JSONObject finalObject = new JSONObject(); + JSONArray posts = new JSONArray(); + + // Iterator is the last object in the post list, let's duplicate it in his own object for clarity. + finalObject.put("iterator", new JSONObject(postsJsonStrings.get(postsJsonStrings.size()-1))); + + for (String postString : postsJsonStrings) { + posts.put(new JSONObject(postString)); + } + finalObject.put("posts", posts); + + if (finalObject.getJSONArray("posts").length() == 1 && !finalObject.getJSONArray("posts").getJSONObject(0).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { + // Only iterator, no posts. + return null; + } + + return finalObject; + + + } catch (URISyntaxException ue) { + // Nothing to catch, it's an hardcoded URI. + } + + return null; + } + + + @Override + protected List getURLsFromJSON(JSONObject json) throws JSONException { + + boolean sortRequested = json.has("posts"); + + int bestArea = 0; + String bestUrl = ""; + List list = new ArrayList<>(); + + JSONArray itemsList = sortRequested ? json.getJSONArray("posts") : json.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").getJSONArray("items"); + + for (Object item : itemsList) { + + if (sortRequested && !((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { + continue; + } + + JSONArray sourcesTMP = sortRequested ? ((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").getJSONArray("mediaSources") : ((JSONObject) item).getJSONArray("mediaSources"); + for (Object sourceTMP : sourcesTMP) + { + int widthTMP = ((JSONObject) sourceTMP).getInt("width"); + int heightTMP = ((JSONObject) sourceTMP).getInt("height"); + int areaTMP = widthTMP * heightTMP; + + if (areaTMP > bestArea) { + bestArea = widthTMP; + bestUrl = ((JSONObject) sourceTMP).getString("url"); + } + } + list.add(bestUrl); + bestUrl = ""; + bestArea = 0; + } + + return list; + } + + @Override + protected JSONObject getFirstPage() throws IOException { + try { + return prepareQuery(null, this.getGID(url), getParameter(url,"sort")); + } catch (URISyntaxException e) { + LOGGER.error(String.format("Error obtaining first page: %s", e.getMessage())); + return null; + } + } + + @Override + public JSONObject getNextPage(JSONObject source) throws IOException { + // Every call the the API contains an "iterator" string that we need to pass to the API to get the next page + // Checking if iterator is null is not working for some reason, hence why the weird "iterator.toString().equals("null")" + + Object iterator = null; + if (source.has("iterator")) { + // Sort requested, custom JSON. + iterator = source.getJSONObject("iterator").getJSONObject("data").getJSONObject("fetchSubreddit").get("iterator"); + } else { + iterator = source.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").get("iterator"); + } + + if (!iterator.toString().equals("null")) { + // Need to change page. + try { + return prepareQuery(iterator.toString(), this.getGID(url), getParameter(url,"sort")); + } catch (URISyntaxException e) { + LOGGER.error(String.format("Error changing page: %s", e.getMessage())); + return null; + } + } else { + return null; + } + } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java index 8e3b9d15..a4fc08cc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VscoRipper.java @@ -1,223 +1,223 @@ -package com.rarchives.ripme.ripper.rippers; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import org.json.JSONObject; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.Connection.Response; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -/** - * For ripping VSCO pictures. - */ -public class VscoRipper extends AbstractHTMLRipper { - - int pageNumber = 1; - JSONObject profileJSON; - - - private static final String DOMAIN = "vsco.co", - HOST = "vsco"; - - public VscoRipper(URL url) throws IOException{ - super(url); - } - - /** - * Checks to see if VscoRipper can Rip specified url. - * @param url - * @return True if can rip. - * False if cannot rip. - */ - @Override - public boolean canRip(URL url) { - if (!url.getHost().endsWith(DOMAIN)) { - return false; - } - // Ignores personalized things (e.g. login, feed) and store page - // Allows links to user profiles and links to images. - //@TODO: Add support for journals and collections. - String u = url.toExternalForm(); - return !u.contains("/store/") || - !u.contains("/feed/") || - !u.contains("/login/") || - !u.contains("/journal/") || - !u.contains("/collection/")|| - !u.contains("/images/") || - u.contains("/media/"); - - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - //no sanitization needed. - return url; - } - - /** - *

Gets the direct URL of full-sized image through the tag.

- * When expanding future functionality (e.g. support from journals), put everything into this method. - * @param page - * @return - */ - @Override - public List getURLsFromPage(Document page){ - List toRip = new ArrayList<>(); - //If user wanted to rip single image - if (url.toString().contains("/media/")){ - try { - toRip.add(vscoImageToURL(url.toExternalForm())); - } catch (IOException ex) { - LOGGER.debug("Failed to convert " + url.toString() + " to external form."); - } - - } else { - String username = getUserName(); - String userTkn = getUserTkn(username); - String siteID = getSiteID(userTkn, username); - while (true) { - profileJSON = getProfileJSON(userTkn, username, Integer.toString(pageNumber), siteID); - for (int i = 0; i < profileJSON.getJSONArray("media").length(); i++) { - toRip.add("https://" + profileJSON.getJSONArray("media").getJSONObject(i).getString("responsive_url")); - } - if (pageNumber * 1000 > profileJSON.getInt("total")) { - return toRip; - } - pageNumber++; - } - - - } - - return toRip; - } - - private String getUserTkn(String username) { - String userTokenPage = "https://vsco.co/content/Static"; - Map responseCookies = new HashMap<>(); - try { - Response resp = Http.url(userTokenPage).ignoreContentType().response(); - responseCookies = resp.cookies(); - return responseCookies.get("vs"); - } catch (IOException e) { - LOGGER.error("Could not get user tkn"); - return null; - } - } - - private String getUserName() { - Pattern p = Pattern.compile("^https?://vsco.co/([a-zA-Z0-9-]+)(/gallery)?(/)?"); - Matcher m = p.matcher(url.toExternalForm()); - - if (m.matches()) { - String user = m.group(1); - return user; - } - return null; - } - - private JSONObject getProfileJSON(String tkn, String username, String page, String siteId) { - String size = "1000"; - String purl = "https://vsco.co/ajxp/" + tkn + "/2.0/medias?site_id=" + siteId + "&page=" + page + "&size=" + size; - Map cookies = new HashMap<>(); - cookies.put("vs", tkn); - try { - JSONObject j = Http.url(purl).cookies(cookies).getJSON(); - return j; - } catch (IOException e) { - LOGGER.error("Could not profile images"); - return null; - } - } - - private String getSiteID(String tkn, String username) { - Map cookies = new HashMap<>(); - cookies.put("vs", tkn); - try { - JSONObject j = Http.url("https://vsco.co/ajxp/" + tkn + "/2.0/sites?subdomain=" + username).cookies(cookies).getJSON(); - return Integer.toString(j.getJSONArray("sites").getJSONObject(0).getInt("id")); - } catch (IOException e) { - LOGGER.error("Could not get site id"); - return null; - } - } - - private String vscoImageToURL(String url) throws IOException{ - Document page = Jsoup.connect(url).userAgent(USER_AGENT) - .get(); - //create Elements filled only with Elements with the "meta" tag. - Elements metaTags = page.getElementsByTag("meta"); - String result = ""; - - for(Element metaTag : metaTags){ - //find URL inside meta-tag with property of "og:image" - if (metaTag.attr("property").equals("og:image")){ - String givenURL = metaTag.attr("content"); - givenURL = givenURL.replaceAll("\\?h=[0-9]+", "");//replace the "?h=xxx" tag at the end of the URL (where each x is a number) - - result = givenURL; - LOGGER.debug("Found image URL: " + givenURL); - break;//immediately stop after getting URL (there should only be 1 image to be downloaded) - } - } - - //Means website changed, things need to be fixed. - if (result.isEmpty()){ - LOGGER.error("Could not find image URL at: " + url); - } - - return result; - - } - - @Override - public String getHost() { - return HOST; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - - //Single Image - Pattern p = Pattern.compile("^https?://vsco\\.co/([a-zA-Z0-9-]+)/media/([a-zA-Z0-9]+)"); - Matcher m = p.matcher(url.toExternalForm()); - - if (m.matches()){ - // Return the text contained between () in the regex - String user = m.group(1); - String imageNum = m.group(2).substring(0, 5);//first 5 characters should be enough to make each rip unique - return user + "/" + imageNum; - } - - //Member profile (Usernames should all be different, so this should work. - p = Pattern.compile("^https?://vsco.co/([a-zA-Z0-9-]+)(/gallery)?(/)?"); - m = p.matcher(url.toExternalForm()); - - if (m.matches()){ - String user = m.group(1); - return user; - } - - throw new MalformedURLException("Expected a URL to a single image or to a member profile, got " + url + " instead"); - - } - - @Override - public String getDomain() { - return DOMAIN; - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } - -} +package com.rarchives.ripme.ripper.rippers; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.json.JSONObject; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.Connection.Response; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +/** + * For ripping VSCO pictures. + */ +public class VscoRipper extends AbstractHTMLRipper { + + int pageNumber = 1; + JSONObject profileJSON; + + + private static final String DOMAIN = "vsco.co", + HOST = "vsco"; + + public VscoRipper(URL url) throws IOException{ + super(url); + } + + /** + * Checks to see if VscoRipper can Rip specified url. + * @param url + * @return True if can rip. + * False if cannot rip. + */ + @Override + public boolean canRip(URL url) { + if (!url.getHost().endsWith(DOMAIN)) { + return false; + } + // Ignores personalized things (e.g. login, feed) and store page + // Allows links to user profiles and links to images. + //@TODO: Add support for journals and collections. + String u = url.toExternalForm(); + return !u.contains("/store/") || + !u.contains("/feed/") || + !u.contains("/login/") || + !u.contains("/journal/") || + !u.contains("/collection/")|| + !u.contains("/images/") || + u.contains("/media/"); + + } + + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + //no sanitization needed. + return url; + } + + /** + *

Gets the direct URL of full-sized image through the tag.

+ * When expanding future functionality (e.g. support from journals), put everything into this method. + * @param page + * @return + */ + @Override + public List getURLsFromPage(Document page){ + List toRip = new ArrayList<>(); + //If user wanted to rip single image + if (url.toString().contains("/media/")){ + try { + toRip.add(vscoImageToURL(url.toExternalForm())); + } catch (IOException ex) { + LOGGER.debug("Failed to convert " + url.toString() + " to external form."); + } + + } else { + String username = getUserName(); + String userTkn = getUserTkn(username); + String siteID = getSiteID(userTkn, username); + while (true) { + profileJSON = getProfileJSON(userTkn, username, Integer.toString(pageNumber), siteID); + for (int i = 0; i < profileJSON.getJSONArray("media").length(); i++) { + toRip.add("https://" + profileJSON.getJSONArray("media").getJSONObject(i).getString("responsive_url")); + } + if (pageNumber * 1000 > profileJSON.getInt("total")) { + return toRip; + } + pageNumber++; + } + + + } + + return toRip; + } + + private String getUserTkn(String username) { + String userTokenPage = "https://vsco.co/content/Static"; + Map responseCookies = new HashMap<>(); + try { + Response resp = Http.url(userTokenPage).ignoreContentType().response(); + responseCookies = resp.cookies(); + return responseCookies.get("vs"); + } catch (IOException e) { + LOGGER.error("Could not get user tkn"); + return null; + } + } + + private String getUserName() { + Pattern p = Pattern.compile("^https?://vsco.co/([a-zA-Z0-9-]+)(/gallery)?(/)?"); + Matcher m = p.matcher(url.toExternalForm()); + + if (m.matches()) { + String user = m.group(1); + return user; + } + return null; + } + + private JSONObject getProfileJSON(String tkn, String username, String page, String siteId) { + String size = "1000"; + String purl = "https://vsco.co/ajxp/" + tkn + "/2.0/medias?site_id=" + siteId + "&page=" + page + "&size=" + size; + Map cookies = new HashMap<>(); + cookies.put("vs", tkn); + try { + JSONObject j = Http.url(purl).cookies(cookies).getJSON(); + return j; + } catch (IOException e) { + LOGGER.error("Could not profile images"); + return null; + } + } + + private String getSiteID(String tkn, String username) { + Map cookies = new HashMap<>(); + cookies.put("vs", tkn); + try { + JSONObject j = Http.url("https://vsco.co/ajxp/" + tkn + "/2.0/sites?subdomain=" + username).cookies(cookies).getJSON(); + return Integer.toString(j.getJSONArray("sites").getJSONObject(0).getInt("id")); + } catch (IOException e) { + LOGGER.error("Could not get site id"); + return null; + } + } + + private String vscoImageToURL(String url) throws IOException{ + Document page = Jsoup.connect(url).userAgent(USER_AGENT) + .get(); + //create Elements filled only with Elements with the "meta" tag. + Elements metaTags = page.getElementsByTag("meta"); + String result = ""; + + for(Element metaTag : metaTags){ + //find URL inside meta-tag with property of "og:image" + if (metaTag.attr("property").equals("og:image")){ + String givenURL = metaTag.attr("content"); + givenURL = givenURL.replaceAll("\\?h=[0-9]+", "");//replace the "?h=xxx" tag at the end of the URL (where each x is a number) + + result = givenURL; + LOGGER.debug("Found image URL: " + givenURL); + break;//immediately stop after getting URL (there should only be 1 image to be downloaded) + } + } + + //Means website changed, things need to be fixed. + if (result.isEmpty()){ + LOGGER.error("Could not find image URL at: " + url); + } + + return result; + + } + + @Override + public String getHost() { + return HOST; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + + //Single Image + Pattern p = Pattern.compile("^https?://vsco\\.co/([a-zA-Z0-9-]+)/media/([a-zA-Z0-9]+)"); + Matcher m = p.matcher(url.toExternalForm()); + + if (m.matches()){ + // Return the text contained between () in the regex + String user = m.group(1); + String imageNum = m.group(2).substring(0, 5);//first 5 characters should be enough to make each rip unique + return user + "/" + imageNum; + } + + //Member profile (Usernames should all be different, so this should work. + p = Pattern.compile("^https?://vsco.co/([a-zA-Z0-9-]+)(/gallery)?(/)?"); + m = p.matcher(url.toExternalForm()); + + if (m.matches()){ + String user = m.group(1); + return user; + } + + throw new MalformedURLException("Expected a URL to a single image or to a member profile, got " + url + " instead"); + + } + + @Override + public String getDomain() { + return DOMAIN; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XlecxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XlecxRipper.java index 0aaacfc4..2e95c04a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XlecxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XlecxRipper.java @@ -1,36 +1,36 @@ -package com.rarchives.ripme.ripper.rippers; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class XlecxRipper extends XcartxRipper { - - private Pattern p = Pattern.compile("^https?://xlecx.org/([a-zA-Z0-9_\\-]+).html"); - - public XlecxRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return "xlecx"; - } - - @Override - public String getDomain() { - return "xlecx.org"; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - throw new MalformedURLException("Expected URL format: http://xlecx.org/comic, got: " + url); - - } -} +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class XlecxRipper extends XcartxRipper { + + private Pattern p = Pattern.compile("^https?://xlecx.org/([a-zA-Z0-9_\\-]+).html"); + + public XlecxRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "xlecx"; + } + + @Override + public String getDomain() { + return "xlecx.org"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected URL format: http://xlecx.org/comic, got: " + url); + + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java index d977708a..c72a5a59 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java @@ -1,80 +1,80 @@ -package com.rarchives.ripme.ripper.rippers.video; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - - -import com.rarchives.ripme.ripper.VideoRipper; -import com.rarchives.ripme.utils.Http; - -public class TwitchVideoRipper extends VideoRipper { - - private static final String HOST = "twitch"; - - public TwitchVideoRipper(URL url) throws IOException { - super(url); - } - - @Override - public String getHost() { - return HOST; - } - - @Override - public boolean canRip(URL url) { - Pattern p = Pattern.compile("^https://clips\\.twitch\\.tv/.*$"); - Matcher m = p.matcher(url.toExternalForm()); - return m.matches(); - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return url; - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https://clips\\.twitch\\.tv/(.*)$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(m.groupCount()); - } - - throw new MalformedURLException( - "Expected Twitch.tv format:" - + "https://clips.twitch.tv/####" - + " Got: " + url); - } - - @Override - public void rip() throws IOException { - LOGGER.info("Retrieving " + this.url); - Document doc = Http.url(url).get(); - - //Get user friendly filename from page title - String title = doc.title(); - - Elements script = doc.select("script"); - if (script.isEmpty()) { - throw new IOException("Could not find script code at " + url); - } - //Regex assumes highest quality source is listed first - Pattern p = Pattern.compile("\"source\":\"(.*?)\""); - - for (Element element : script) { - Matcher m = p.matcher(element.data()); - if (m.find()){ - String vidUrl = m.group(1); - addURLToDownload(new URL(vidUrl), HOST + "_" + title); - } - } - waitForThreads(); - } +package com.rarchives.ripme.ripper.rippers.video; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + + +import com.rarchives.ripme.ripper.VideoRipper; +import com.rarchives.ripme.utils.Http; + +public class TwitchVideoRipper extends VideoRipper { + + private static final String HOST = "twitch"; + + public TwitchVideoRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return HOST; + } + + @Override + public boolean canRip(URL url) { + Pattern p = Pattern.compile("^https://clips\\.twitch\\.tv/.*$"); + Matcher m = p.matcher(url.toExternalForm()); + return m.matches(); + } + + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + return url; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^https://clips\\.twitch\\.tv/(.*)$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(m.groupCount()); + } + + throw new MalformedURLException( + "Expected Twitch.tv format:" + + "https://clips.twitch.tv/####" + + " Got: " + url); + } + + @Override + public void rip() throws IOException { + LOGGER.info("Retrieving " + this.url); + Document doc = Http.url(url).get(); + + //Get user friendly filename from page title + String title = doc.title(); + + Elements script = doc.select("script"); + if (script.isEmpty()) { + throw new IOException("Could not find script code at " + url); + } + //Regex assumes highest quality source is listed first + Pattern p = Pattern.compile("\"source\":\"(.*?)\""); + + for (Element element : script) { + Matcher m = p.matcher(element.data()); + if (m.find()){ + String vidUrl = m.group(1); + addURLToDownload(new URL(vidUrl), HOST + "_" + title); + } + } + waitForThreads(); + } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/utils/Proxy.java b/src/main/java/com/rarchives/ripme/utils/Proxy.java index be3c3b7e..0275bd5c 100644 --- a/src/main/java/com/rarchives/ripme/utils/Proxy.java +++ b/src/main/java/com/rarchives/ripme/utils/Proxy.java @@ -1,99 +1,99 @@ -package com.rarchives.ripme.utils; - -import java.net.Authenticator; -import java.net.PasswordAuthentication; -import java.util.Map; -import java.util.HashMap; - -/** - * Proxy/Socks setter - */ -public class Proxy { - private Proxy() { - } - - /** - * Parse the proxy server settings from string, using the format - * [user:password]@host[:port]. - * - * @param fullproxy the string to parse - * @return HashMap containing proxy server, port, user and password - */ - private static Map parseServer(String fullproxy) { - Map proxy = new HashMap(); - - if (fullproxy.lastIndexOf("@") != -1) { - int sservli = fullproxy.lastIndexOf("@"); - String userpw = fullproxy.substring(0, sservli); - String[] usersplit = userpw.split(":"); - proxy.put("user", usersplit[0]); - proxy.put("password", usersplit[1]); - fullproxy = fullproxy.substring(sservli + 1); - } - String[] servsplit = fullproxy.split(":"); - if (servsplit.length == 2) { - proxy.put("port", servsplit[1]); - } - proxy.put("server", servsplit[0]); - return proxy; - } - - /** - * Set a HTTP Proxy. - * WARNING: Authenticated HTTP Proxy won't work from jdk1.8.111 unless - * passing the flag -Djdk.http.auth.tunneling.disabledSchemes="" to java - * see https://stackoverflow.com/q/41505219 - * - * @param fullproxy the proxy, using format [user:password]@host[:port] - */ - public static void setHTTPProxy(String fullproxy) { - Map proxyServer = parseServer(fullproxy); - - if (proxyServer.get("user") != null && proxyServer.get("password") != null) { - Authenticator.setDefault(new Authenticator(){ - protected PasswordAuthentication getPasswordAuthentication(){ - PasswordAuthentication p = new PasswordAuthentication(proxyServer.get("user"), proxyServer.get("password").toCharArray()); - return p; - } - }); - System.setProperty("http.proxyUser", proxyServer.get("user")); - System.setProperty("http.proxyPassword", proxyServer.get("password")); - System.setProperty("https.proxyUser", proxyServer.get("user")); - System.setProperty("https.proxyPassword", proxyServer.get("password")); - } - - if (proxyServer.get("port") != null) { - System.setProperty("http.proxyPort", proxyServer.get("port")); - System.setProperty("https.proxyPort", proxyServer.get("port")); - } - - System.setProperty("http.proxyHost", proxyServer.get("server")); - System.setProperty("https.proxyHost", proxyServer.get("server")); - } - - /** - * Set a Socks Proxy Server (globally). - * - * @param fullsocks the socks server, using format [user:password]@host[:port] - */ - public static void setSocks(String fullsocks) { - - Map socksServer = parseServer(fullsocks); - if (socksServer.get("user") != null && socksServer.get("password") != null) { - Authenticator.setDefault(new Authenticator(){ - protected PasswordAuthentication getPasswordAuthentication(){ - PasswordAuthentication p = new PasswordAuthentication(socksServer.get("user"), socksServer.get("password").toCharArray()); - return p; - } - }); - System.setProperty("java.net.socks.username", socksServer.get("user")); - System.setProperty("java.net.socks.password", socksServer.get("password")); - } - if (socksServer.get("port") != null) { - System.setProperty("socksProxyPort", socksServer.get("port")); - } - - System.setProperty("socksProxyHost", socksServer.get("server")); - } - -} +package com.rarchives.ripme.utils; + +import java.net.Authenticator; +import java.net.PasswordAuthentication; +import java.util.Map; +import java.util.HashMap; + +/** + * Proxy/Socks setter + */ +public class Proxy { + private Proxy() { + } + + /** + * Parse the proxy server settings from string, using the format + * [user:password]@host[:port]. + * + * @param fullproxy the string to parse + * @return HashMap containing proxy server, port, user and password + */ + private static Map parseServer(String fullproxy) { + Map proxy = new HashMap(); + + if (fullproxy.lastIndexOf("@") != -1) { + int sservli = fullproxy.lastIndexOf("@"); + String userpw = fullproxy.substring(0, sservli); + String[] usersplit = userpw.split(":"); + proxy.put("user", usersplit[0]); + proxy.put("password", usersplit[1]); + fullproxy = fullproxy.substring(sservli + 1); + } + String[] servsplit = fullproxy.split(":"); + if (servsplit.length == 2) { + proxy.put("port", servsplit[1]); + } + proxy.put("server", servsplit[0]); + return proxy; + } + + /** + * Set a HTTP Proxy. + * WARNING: Authenticated HTTP Proxy won't work from jdk1.8.111 unless + * passing the flag -Djdk.http.auth.tunneling.disabledSchemes="" to java + * see https://stackoverflow.com/q/41505219 + * + * @param fullproxy the proxy, using format [user:password]@host[:port] + */ + public static void setHTTPProxy(String fullproxy) { + Map proxyServer = parseServer(fullproxy); + + if (proxyServer.get("user") != null && proxyServer.get("password") != null) { + Authenticator.setDefault(new Authenticator(){ + protected PasswordAuthentication getPasswordAuthentication(){ + PasswordAuthentication p = new PasswordAuthentication(proxyServer.get("user"), proxyServer.get("password").toCharArray()); + return p; + } + }); + System.setProperty("http.proxyUser", proxyServer.get("user")); + System.setProperty("http.proxyPassword", proxyServer.get("password")); + System.setProperty("https.proxyUser", proxyServer.get("user")); + System.setProperty("https.proxyPassword", proxyServer.get("password")); + } + + if (proxyServer.get("port") != null) { + System.setProperty("http.proxyPort", proxyServer.get("port")); + System.setProperty("https.proxyPort", proxyServer.get("port")); + } + + System.setProperty("http.proxyHost", proxyServer.get("server")); + System.setProperty("https.proxyHost", proxyServer.get("server")); + } + + /** + * Set a Socks Proxy Server (globally). + * + * @param fullsocks the socks server, using format [user:password]@host[:port] + */ + public static void setSocks(String fullsocks) { + + Map socksServer = parseServer(fullsocks); + if (socksServer.get("user") != null && socksServer.get("password") != null) { + Authenticator.setDefault(new Authenticator(){ + protected PasswordAuthentication getPasswordAuthentication(){ + PasswordAuthentication p = new PasswordAuthentication(socksServer.get("user"), socksServer.get("password").toCharArray()); + return p; + } + }); + System.setProperty("java.net.socks.username", socksServer.get("user")); + System.setProperty("java.net.socks.password", socksServer.get("password")); + } + if (socksServer.get("port") != null) { + System.setProperty("socksProxyPort", socksServer.get("port")); + } + + System.setProperty("socksProxyHost", socksServer.get("server")); + } + +} diff --git a/src/main/resources/LabelsBundle_pl_PL.properties b/src/main/resources/LabelsBundle_pl_PL.properties index 4ba4590e..dbb74ef1 100644 --- a/src/main/resources/LabelsBundle_pl_PL.properties +++ b/src/main/resources/LabelsBundle_pl_PL.properties @@ -1,59 +1,59 @@ -Log = Logi -History = Historia -created = Stworzono -modified = Zmodyfikowano -queue = Kolejka -Configuration = Konfiguracja - -# Keys for the Configuration menu - -current.version = Obecna Wersja -check.for.updates = Sprawdź dostępność aktualizacji -auto.update = Auto Aktualizacja? -max.download.threads = Maksymalna Ilośc Pobieranych Plików: -timeout.mill = Opóźnienie (w milisekundach): -retry.download.count = Liczba ponownych pobrań -overwrite.existing.files = Nadpisać istniejące pliki? -sound.when.rip.completes = Dźwięk po zakończeniu -preserve.order = Zachować porządek -save.logs = Zapisz Logi -notification.when.rip.starts = Powiadomienie przy uruchomieniu pobierania -save.urls.only = Zapisz tylko linki -save.album.titles = Zapisz nazwy albumów -autorip.from.clipboard = Auto pobieranie ze schowka -save.descriptions = Zapisz opis -prefer.mp4.over.gif = Preferuj MP4 od GIF -restore.window.position = Przywróć pozycję okna -remember.url.history = Zapamiętaj historię linków -loading.history.from = Załaduj historię z... - -# Misc UI keys - -loading.history.from.configuration = Załaduj historię z ustawień -interrupted.while.waiting.to.rip.next.album = Przerwany podczas oczekiwania na zgrywanie następnego albumu -inactive = Nieaktywny -re-rip.checked = Sprawdź pobrane ripy -remove = Usuń -clear = Wyczyść -download.url.list = Pobierz listę linków -select.save.dir = Wybierz ścieżkę zapisu - -# Keys for the logs generated by DownloadFileThread - -nonretriable.status.code = Nieodwracalny kod statusu -retriable.status.code = Odzyskiwanie kodu statusu -server.doesnt.support.resuming.downloads = Serwer nie obsługuje wznowienia pobierania - -# A "magic number" can also be called a file signature - -was.unable.to.get.content.type.using.magic.number = Nie udało się uzyskać typu zawartości za pomocą magicznej liczby -magic.number.was = Magiczną liczbą była -deleting.existing.file = Usuwanie istniejących plików -request.properties = Poproś o uprawnienia -download.interrupted = Pobieranie przerwane -exceeded.maximum.retries = Spodziewana ilośc powtórzeń -http.status.exception = Wyjątek statusu http -exception.while.downloading.file = Wystąpił problem podczas pobierania pliku -failed.to.download = Nie można pobrać pliku -skipping = Pomijanie +Log = Logi +History = Historia +created = Stworzono +modified = Zmodyfikowano +queue = Kolejka +Configuration = Konfiguracja + +# Keys for the Configuration menu + +current.version = Obecna Wersja +check.for.updates = Sprawdź dostępność aktualizacji +auto.update = Auto Aktualizacja? +max.download.threads = Maksymalna Ilośc Pobieranych Plików: +timeout.mill = Opóźnienie (w milisekundach): +retry.download.count = Liczba ponownych pobrań +overwrite.existing.files = Nadpisać istniejące pliki? +sound.when.rip.completes = Dźwięk po zakończeniu +preserve.order = Zachować porządek +save.logs = Zapisz Logi +notification.when.rip.starts = Powiadomienie przy uruchomieniu pobierania +save.urls.only = Zapisz tylko linki +save.album.titles = Zapisz nazwy albumów +autorip.from.clipboard = Auto pobieranie ze schowka +save.descriptions = Zapisz opis +prefer.mp4.over.gif = Preferuj MP4 od GIF +restore.window.position = Przywróć pozycję okna +remember.url.history = Zapamiętaj historię linków +loading.history.from = Załaduj historię z... + +# Misc UI keys + +loading.history.from.configuration = Załaduj historię z ustawień +interrupted.while.waiting.to.rip.next.album = Przerwany podczas oczekiwania na zgrywanie następnego albumu +inactive = Nieaktywny +re-rip.checked = Sprawdź pobrane ripy +remove = Usuń +clear = Wyczyść +download.url.list = Pobierz listę linków +select.save.dir = Wybierz ścieżkę zapisu + +# Keys for the logs generated by DownloadFileThread + +nonretriable.status.code = Nieodwracalny kod statusu +retriable.status.code = Odzyskiwanie kodu statusu +server.doesnt.support.resuming.downloads = Serwer nie obsługuje wznowienia pobierania + +# A "magic number" can also be called a file signature + +was.unable.to.get.content.type.using.magic.number = Nie udało się uzyskać typu zawartości za pomocą magicznej liczby +magic.number.was = Magiczną liczbą była +deleting.existing.file = Usuwanie istniejących plików +request.properties = Poproś o uprawnienia +download.interrupted = Pobieranie przerwane +exceeded.maximum.retries = Spodziewana ilośc powtórzeń +http.status.exception = Wyjątek statusu http +exception.while.downloading.file = Wystąpił problem podczas pobierania pliku +failed.to.download = Nie można pobrać pliku +skipping = Pomijanie file.already.exists = Plik już istnieje \ No newline at end of file diff --git a/src/main/resources/LabelsBundle_zh_CN.properties b/src/main/resources/LabelsBundle_zh_CN.properties index 994efcf8..cd43da1c 100644 --- a/src/main/resources/LabelsBundle_zh_CN.properties +++ b/src/main/resources/LabelsBundle_zh_CN.properties @@ -1,75 +1,75 @@ -Log = 日志 -History = 历史 -created = 创建时间 -modified = 修改时间 -queue = 队列 -Configuration = 配置 -open = 打开 - -# Keys for the Configuration menu -current.version = 当前版本 -check.for.updates = 检查更新 -auto.update = 自动更新？ -max.download.threads = 最大下载线程数： -timeout.mill = 超时（毫秒）： -retry.download.count = 重试下载次数 -overwrite.existing.files = 覆盖现有文件？ -sound.when.rip.completes = 抓取完成时播放声音 -preserve.order = 保持顺序 -save.logs = 保存日志 -notification.when.rip.starts = 通知抓取开始 -save.urls.only = 仅保存 URL -save.album.titles = 保存专辑标题 -autorip.from.clipboard = 监视剪贴板上的 URL -save.descriptions = 保存描述 -prefer.mp4.over.gif = 首选 MP4 而非 GIF -restore.window.position = 恢复窗口位置 -remember.url.history = 记住 URL 历史 -loading.history.from = 加载历史从 - -# Queue keys -queue.remove.all = 移除全部 -queue.validation = 您确定要移除队列内的全部项目？ -queue.remove.selected = 移除所选项目 - -# History -re-rip.checked = 重新抓取选中的项目 -remove = 移除 -clear = 清除 -history.check.all = 选中全部 -history.check.none = 取消选中全部 -history.check.selected = 选中所选项目 -history.uncheck.selected = 取消选中所选项目 -history.load.failed.warning = RipMe 加载位于 historyFile.getAbsolutePath() 的历史文件失败\n\n错误：%s\n\n关闭 RipMe 会自动覆盖此文件的内容，\n请在关闭 RipMe 前备份它！ -history.load.none = 无可重新抓取的历史条目。请先抓取一些专辑 -history.load.none.checked = 未 '选中' 任何历史条目，请通过选中所需 URL 前面的复选框或URL 的右键菜单以选中所需条目 - -# TrayIcon -tray.show = 显示 -tray.hide = 隐藏 -tray.autorip = 监视剪贴板上的 URL -tray.exit = 退出 - -# Misc UI keys -loading.history.from.configuration = 从配置加载历史 -interrupted.while.waiting.to.rip.next.album = 等候抓取下一专辑期间发生中断 -inactive = 非活动 -download.url.list = 下载 URL 列表 -select.save.dir = 选择保存目录 - -# Keys for the logs generated by DownloadFileThread -nonretriable.status.code = 非可重试状态代码 -retriable.status.code = 可重试状态代码 -server.doesnt.support.resuming.downloads = 服务器不支持继续下载（续传） -# A "magic number" can also be called a file signature -was.unable.to.get.content.type.using.magic.number = 不能使用幻数获取内容类型 -magic.number.was = 幻数为 -deleting.existing.file = 删除现有文件 -request.properties = 请求属性 -download.interrupted = 下载中断 -exceeded.maximum.retries = 超过最大重试次数 -http.status.exception = HTTP 状态意外 -exception.while.downloading.file = 下载文件时发生意外 -failed.to.download = 下载失败 -skipping = 跳过 +Log = 日志 +History = 历史 +created = 创建时间 +modified = 修改时间 +queue = 队列 +Configuration = 配置 +open = 打开 + +# Keys for the Configuration menu +current.version = 当前版本 +check.for.updates = 检查更新 +auto.update = 自动更新？ +max.download.threads = 最大下载线程数： +timeout.mill = 超时（毫秒）： +retry.download.count = 重试下载次数 +overwrite.existing.files = 覆盖现有文件？ +sound.when.rip.completes = 抓取完成时播放声音 +preserve.order = 保持顺序 +save.logs = 保存日志 +notification.when.rip.starts = 通知抓取开始 +save.urls.only = 仅保存 URL +save.album.titles = 保存专辑标题 +autorip.from.clipboard = 监视剪贴板上的 URL +save.descriptions = 保存描述 +prefer.mp4.over.gif = 首选 MP4 而非 GIF +restore.window.position = 恢复窗口位置 +remember.url.history = 记住 URL 历史 +loading.history.from = 加载历史从 + +# Queue keys +queue.remove.all = 移除全部 +queue.validation = 您确定要移除队列内的全部项目？ +queue.remove.selected = 移除所选项目 + +# History +re-rip.checked = 重新抓取选中的项目 +remove = 移除 +clear = 清除 +history.check.all = 选中全部 +history.check.none = 取消选中全部 +history.check.selected = 选中所选项目 +history.uncheck.selected = 取消选中所选项目 +history.load.failed.warning = RipMe 加载位于 historyFile.getAbsolutePath() 的历史文件失败\n\n错误：%s\n\n关闭 RipMe 会自动覆盖此文件的内容，\n请在关闭 RipMe 前备份它！ +history.load.none = 无可重新抓取的历史条目。请先抓取一些专辑 +history.load.none.checked = 未 '选中' 任何历史条目，请通过选中所需 URL 前面的复选框或URL 的右键菜单以选中所需条目 + +# TrayIcon +tray.show = 显示 +tray.hide = 隐藏 +tray.autorip = 监视剪贴板上的 URL +tray.exit = 退出 + +# Misc UI keys +loading.history.from.configuration = 从配置加载历史 +interrupted.while.waiting.to.rip.next.album = 等候抓取下一专辑期间发生中断 +inactive = 非活动 +download.url.list = 下载 URL 列表 +select.save.dir = 选择保存目录 + +# Keys for the logs generated by DownloadFileThread +nonretriable.status.code = 非可重试状态代码 +retriable.status.code = 可重试状态代码 +server.doesnt.support.resuming.downloads = 服务器不支持继续下载（续传） +# A "magic number" can also be called a file signature +was.unable.to.get.content.type.using.magic.number = 不能使用幻数获取内容类型 +magic.number.was = 幻数为 +deleting.existing.file = 删除现有文件 +request.properties = 请求属性 +download.interrupted = 下载中断 +exceeded.maximum.retries = 超过最大重试次数 +http.status.exception = HTTP 状态意外 +exception.while.downloading.file = 下载文件时发生意外 +failed.to.download = 下载失败 +skipping = 跳过 file.already.exists = 文件已存在 \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/proxyTest.java b/src/test/java/com/rarchives/ripme/tst/proxyTest.java index 7791290a..0576b8e2 100644 --- a/src/test/java/com/rarchives/ripme/tst/proxyTest.java +++ b/src/test/java/com/rarchives/ripme/tst/proxyTest.java @@ -1,57 +1,57 @@ -package com.rarchives.ripme.tst; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import com.rarchives.ripme.utils.Proxy; -import com.rarchives.ripme.utils.Utils; -import com.rarchives.ripme.utils.Http; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertFalse; - -public class proxyTest { - - - // This test will only run on machines where the user has added a entry for proxy.socks - @Test - public void testSocksProxy() throws IOException, URISyntaxException { - // Unset proxy before testing - System.setProperty("http.proxyHost", ""); - System.setProperty("https.proxyHost", ""); - System.setProperty("socksProxyHost", ""); - URL url = new URI("https://icanhazip.com").toURL(); - String proxyConfig = Utils.getConfigString("proxy.socks", ""); - if (!proxyConfig.equals("")) { - String ip1 = Http.url(url).ignoreContentType().get().text(); - Proxy.setSocks(Utils.getConfigString("proxy.socks", "")); - String ip2 = Http.url(url).ignoreContentType().get().text(); - assertFalse(ip1.equals(ip2)); - } else { - System.out.println("Skipping testSocksProxy"); - assert(true); - } - } - - // This test will only run on machines where the user has added a entry for proxy.http - @Test - public void testHTTPProxy() throws IOException, URISyntaxException { - // Unset proxy before testing - System.setProperty("http.proxyHost", ""); - System.setProperty("https.proxyHost", ""); - System.setProperty("socksProxyHost", ""); - URL url = new URI("https://icanhazip.com").toURL(); - String proxyConfig = Utils.getConfigString("proxy.http", ""); - if (!proxyConfig.equals("")) { - String ip1 = Http.url(url).ignoreContentType().get().text(); - Proxy.setHTTPProxy(Utils.getConfigString("proxy.http", "")); - String ip2 = Http.url(url).ignoreContentType().get().text(); - assertFalse(ip1.equals(ip2)); - } else { - System.out.println("Skipping testHTTPProxy"); - assert(true); - } - } - -} +package com.rarchives.ripme.tst; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import com.rarchives.ripme.utils.Proxy; +import com.rarchives.ripme.utils.Utils; +import com.rarchives.ripme.utils.Http; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; + +public class proxyTest { + + + // This test will only run on machines where the user has added a entry for proxy.socks + @Test + public void testSocksProxy() throws IOException, URISyntaxException { + // Unset proxy before testing + System.setProperty("http.proxyHost", ""); + System.setProperty("https.proxyHost", ""); + System.setProperty("socksProxyHost", ""); + URL url = new URI("https://icanhazip.com").toURL(); + String proxyConfig = Utils.getConfigString("proxy.socks", ""); + if (!proxyConfig.equals("")) { + String ip1 = Http.url(url).ignoreContentType().get().text(); + Proxy.setSocks(Utils.getConfigString("proxy.socks", "")); + String ip2 = Http.url(url).ignoreContentType().get().text(); + assertFalse(ip1.equals(ip2)); + } else { + System.out.println("Skipping testSocksProxy"); + assert(true); + } + } + + // This test will only run on machines where the user has added a entry for proxy.http + @Test + public void testHTTPProxy() throws IOException, URISyntaxException { + // Unset proxy before testing + System.setProperty("http.proxyHost", ""); + System.setProperty("https.proxyHost", ""); + System.setProperty("socksProxyHost", ""); + URL url = new URI("https://icanhazip.com").toURL(); + String proxyConfig = Utils.getConfigString("proxy.http", ""); + if (!proxyConfig.equals("")) { + String ip1 = Http.url(url).ignoreContentType().get().text(); + Proxy.setHTTPProxy(Utils.getConfigString("proxy.http", "")); + String ip2 = Http.url(url).ignoreContentType().get().text(); + assertFalse(ip1.equals(ip2)); + } else { + System.out.println("Skipping testHTTPProxy"); + assert(true); + } + } + +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java index 0a64487d..e01ae6e0 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java @@ -1,28 +1,28 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import com.rarchives.ripme.ripper.rippers.ComicextraRipper; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -public class ComicextraRipperTest extends RippersTest { - @Test - @Tag("flaky") - public void testComicUrl() throws IOException, URISyntaxException { - URL url = new URI("https://www.comicextra.com/comic/karma-police").toURL(); - ComicextraRipper ripper = new ComicextraRipper(url); - testRipper(ripper); - } - @Test - @Disabled("no images found error, broken ripper?") - public void testChapterUrl() throws IOException, URISyntaxException { - URL url = new URI("https://www.comicextra.com/v-for-vendetta/chapter-1").toURL(); - ComicextraRipper ripper = new ComicextraRipper(url); - testRipper(ripper); - } - -} +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import com.rarchives.ripme.ripper.rippers.ComicextraRipper; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +public class ComicextraRipperTest extends RippersTest { + @Test + @Tag("flaky") + public void testComicUrl() throws IOException, URISyntaxException { + URL url = new URI("https://www.comicextra.com/comic/karma-police").toURL(); + ComicextraRipper ripper = new ComicextraRipper(url); + testRipper(ripper); + } + @Test + @Disabled("no images found error, broken ripper?") + public void testChapterUrl() throws IOException, URISyntaxException { + URL url = new URI("https://www.comicextra.com/v-for-vendetta/chapter-1").toURL(); + ComicextraRipper ripper = new ComicextraRipper(url); + testRipper(ripper); + } + +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java index 17f0c8a6..14fcef07 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java @@ -1,55 +1,55 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import com.rarchives.ripme.ripper.rippers.CyberdropRipper; -import com.rarchives.ripme.utils.Http; -import org.jsoup.nodes.Document; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class CyberdropRipperTest extends RippersTest { - @Test - public void testScrolllerGID() throws IOException, URISyntaxException { - Map testURLs = new HashMap<>(); - - testURLs.put(new URI("https://cyberdrop.me/a/n4umdBjw").toURL(), "n4umdBjw"); - testURLs.put(new URI("https://cyberdrop.me/a/iLtp4BjW").toURL(), "iLtp4BjW"); - for (URL url : testURLs.keySet()) { - CyberdropRipper ripper = new CyberdropRipper(url); - ripper.setup(); - Assertions.assertEquals(testURLs.get(url), ripper.getGID(ripper.getURL())); - deleteDir(ripper.getWorkingDir()); - } - } - - @Test - @Tag("flaky") - public void testCyberdropNumberOfFiles() throws IOException, URISyntaxException { - List testURLs = new ArrayList(); - - testURLs.add(new URI("https://cyberdrop.me/a/n4umdBjw").toURL()); - testURLs.add(new URI("https://cyberdrop.me/a/iLtp4BjW").toURL()); - for (URL url : testURLs) { - Assertions.assertTrue(willDownloadAllFiles(url)); - } - } - - public boolean willDownloadAllFiles(URL url) throws IOException { - Document doc = Http.url(url).get(); - long numberOfLinks = doc.getElementsByClass("image").stream().count(); - int numberOfFiles = Integer.parseInt(doc.getElementById("totalFilesAmount").text()); - return numberOfLinks == numberOfFiles; - } - - - +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.CyberdropRipper; +import com.rarchives.ripme.utils.Http; +import org.jsoup.nodes.Document; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class CyberdropRipperTest extends RippersTest { + @Test + public void testScrolllerGID() throws IOException, URISyntaxException { + Map testURLs = new HashMap<>(); + + testURLs.put(new URI("https://cyberdrop.me/a/n4umdBjw").toURL(), "n4umdBjw"); + testURLs.put(new URI("https://cyberdrop.me/a/iLtp4BjW").toURL(), "iLtp4BjW"); + for (URL url : testURLs.keySet()) { + CyberdropRipper ripper = new CyberdropRipper(url); + ripper.setup(); + Assertions.assertEquals(testURLs.get(url), ripper.getGID(ripper.getURL())); + deleteDir(ripper.getWorkingDir()); + } + } + + @Test + @Tag("flaky") + public void testCyberdropNumberOfFiles() throws IOException, URISyntaxException { + List testURLs = new ArrayList(); + + testURLs.add(new URI("https://cyberdrop.me/a/n4umdBjw").toURL()); + testURLs.add(new URI("https://cyberdrop.me/a/iLtp4BjW").toURL()); + for (URL url : testURLs) { + Assertions.assertTrue(willDownloadAllFiles(url)); + } + } + + public boolean willDownloadAllFiles(URL url) throws IOException { + Document doc = Http.url(url).get(); + long numberOfLinks = doc.getElementsByClass("image").stream().count(); + int numberOfFiles = Integer.parseInt(doc.getElementById("totalFilesAmount").text()); + return numberOfLinks == numberOfFiles; + } + + + } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java index cc418842..d4a51a68 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FolioRipperTest.java @@ -1,30 +1,30 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import com.rarchives.ripme.ripper.rippers.FolioRipper; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -public class FolioRipperTest extends RippersTest { - /** - * Test for folio.ink ripper - */ - @Test - @Disabled("test or ripper broken") - public void testFolioRip() throws IOException, URISyntaxException { - FolioRipper ripper = new FolioRipper(new URI("https://folio.ink/DmBe6i").toURL()); - testRipper(ripper); - } - - @Test - public void testGetGID() throws IOException, URISyntaxException { - URL url = new URI("https://folio.ink/DmBe6i").toURL(); - FolioRipper ripper = new FolioRipper(url); - Assertions.assertEquals("DmBe6i", ripper.getGID(url)); - } -} +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import com.rarchives.ripme.ripper.rippers.FolioRipper; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +public class FolioRipperTest extends RippersTest { + /** + * Test for folio.ink ripper + */ + @Test + @Disabled("test or ripper broken") + public void testFolioRip() throws IOException, URISyntaxException { + FolioRipper ripper = new FolioRipper(new URI("https://folio.ink/DmBe6i").toURL()); + testRipper(ripper); + } + + @Test + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://folio.ink/DmBe6i").toURL(); + FolioRipper ripper = new FolioRipper(url); + Assertions.assertEquals("DmBe6i", ripper.getGID(url)); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java index ac06ceb6..c8400d56 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java @@ -1,53 +1,53 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import com.rarchives.ripme.ripper.rippers.GfycatRipper; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - - -public class GfycatRipperTest extends RippersTest { - - /** - * Rips correctly formatted URL directly from Gfycat - */ - @Test - public void testGfycatGoodURL() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/TemptingExcellentIchthyosaurs").toURL()); - testRipper(ripper); - } - /** - * Rips badly formatted URL directly from Gfycat - */ - public void testGfycatBadURL() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/gifs/detail/limitedtestyamericancrow").toURL()); - testRipper(ripper); - } - - /** - * Rips a Gfycat profile - */ - public void testGfycatProfile() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/@golbanstorage").toURL()); - testRipper(ripper); - } - - /** - * Rips a Gfycat amp link - * @throws IOException - */ - public void testGfycatAmp() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/amp/TemptingExcellentIchthyosaurs").toURL()); - testRipper(ripper); - } - - /** - * Rips a Gfycat profile with special characters in username - */ - public void testGfycatSpecialChar() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/@rsss.kr").toURL()); - testRipper(ripper); - } -} +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.GfycatRipper; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + + +public class GfycatRipperTest extends RippersTest { + + /** + * Rips correctly formatted URL directly from Gfycat + */ + @Test + public void testGfycatGoodURL() throws IOException, URISyntaxException { + GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/TemptingExcellentIchthyosaurs").toURL()); + testRipper(ripper); + } + /** + * Rips badly formatted URL directly from Gfycat + */ + public void testGfycatBadURL() throws IOException, URISyntaxException { + GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/gifs/detail/limitedtestyamericancrow").toURL()); + testRipper(ripper); + } + + /** + * Rips a Gfycat profile + */ + public void testGfycatProfile() throws IOException, URISyntaxException { + GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/@golbanstorage").toURL()); + testRipper(ripper); + } + + /** + * Rips a Gfycat amp link + * @throws IOException + */ + public void testGfycatAmp() throws IOException, URISyntaxException { + GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/amp/TemptingExcellentIchthyosaurs").toURL()); + testRipper(ripper); + } + + /** + * Rips a Gfycat profile with special characters in username + */ + public void testGfycatSpecialChar() throws IOException, URISyntaxException { + GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/@rsss.kr").toURL()); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java index ca3aee41..4516f2c8 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java @@ -1,42 +1,42 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - -import com.rarchives.ripme.ripper.rippers.ListalRipper; -import org.junit.jupiter.api.Test; - -public class ListalRipperTest extends RippersTest { - - /** - * Test for list type url. - */ - @Test - public void testPictures() throws IOException, URISyntaxException { - ListalRipper ripper = - new ListalRipper(new URI("https://www.listal.com/emma-stone_iii/pictures").toURL()); - testRipper(ripper); - } - - /** - * Test for list type url. - */ - @Test - public void testRipListType() throws IOException, URISyntaxException { - ListalRipper ripper = - new ListalRipper(new URI("https://www.listal.com/list/evolution-emma-stone").toURL()); - testRipper(ripper); - } - - /** - * Test for folder type url. - */ - @Test - public void testRipFolderType() throws IOException, URISyntaxException { - ListalRipper ripper = - new ListalRipper(new URI("https://www.listal.com/chet-atkins/pictures").toURL()); - testRipper(ripper); - } - -} +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +import com.rarchives.ripme.ripper.rippers.ListalRipper; +import org.junit.jupiter.api.Test; + +public class ListalRipperTest extends RippersTest { + + /** + * Test for list type url. + */ + @Test + public void testPictures() throws IOException, URISyntaxException { + ListalRipper ripper = + new ListalRipper(new URI("https://www.listal.com/emma-stone_iii/pictures").toURL()); + testRipper(ripper); + } + + /** + * Test for list type url. + */ + @Test + public void testRipListType() throws IOException, URISyntaxException { + ListalRipper ripper = + new ListalRipper(new URI("https://www.listal.com/list/evolution-emma-stone").toURL()); + testRipper(ripper); + } + + /** + * Test for folder type url. + */ + @Test + public void testRipFolderType() throws IOException, URISyntaxException { + ListalRipper ripper = + new ListalRipper(new URI("https://www.listal.com/chet-atkins/pictures").toURL()); + testRipper(ripper); + } + +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java index f2b7461b..44bf06cf 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java @@ -1,55 +1,55 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import com.rarchives.ripme.ripper.rippers.ScrolllerRipper; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.HashMap; -import java.util.Map; - -public class ScrolllerRipperTest extends RippersTest { - @Test - public void testScrolllerGID() throws IOException, URISyntaxException { - Map testURLs = new HashMap<>(); - - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp").toURL(), "CatsStandingUp"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures").toURL(), "CatsStandingUp"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=pictures").toURL(), "CatsStandingUp"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures&sort=top").toURL(), "CatsStandingUp"); - for (URL url : testURLs.keySet()) { - ScrolllerRipper ripper = new ScrolllerRipper(url); - ripper.setup(); - Assertions.assertEquals(testURLs.get(url), ripper.getGID(ripper.getURL())); - deleteDir(ripper.getWorkingDir()); - } - } - - @Test - public void testScrolllerFilterRegex() throws IOException, URISyntaxException { - Map testURLs = new HashMap<>(); - - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp").toURL(), "NOFILTER"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures").toURL(), "PICTURE"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=videos").toURL(), "VIDEO"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=albums").toURL(), "ALBUM"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=pictures").toURL(), "PICTURE"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=videos").toURL(), "VIDEO"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=albums").toURL(), "ALBUM"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures&sort=top").toURL(), "PICTURE"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=videos&sort=top").toURL(), "VIDEO"); - testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=albums&sort=top").toURL(), "ALBUM"); - for (URL url : testURLs.keySet()) { - ScrolllerRipper ripper = new ScrolllerRipper(url); - ripper.setup(); - Assertions.assertEquals(testURLs.get(url), ripper.convertFilterString(ripper.getParameter(ripper.getURL(),"filter"))); - deleteDir(ripper.getWorkingDir()); - } - } - - - -} +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.ScrolllerRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; + +public class ScrolllerRipperTest extends RippersTest { + @Test + public void testScrolllerGID() throws IOException, URISyntaxException { + Map testURLs = new HashMap<>(); + + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp").toURL(), "CatsStandingUp"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures").toURL(), "CatsStandingUp"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=pictures").toURL(), "CatsStandingUp"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures&sort=top").toURL(), "CatsStandingUp"); + for (URL url : testURLs.keySet()) { + ScrolllerRipper ripper = new ScrolllerRipper(url); + ripper.setup(); + Assertions.assertEquals(testURLs.get(url), ripper.getGID(ripper.getURL())); + deleteDir(ripper.getWorkingDir()); + } + } + + @Test + public void testScrolllerFilterRegex() throws IOException, URISyntaxException { + Map testURLs = new HashMap<>(); + + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp").toURL(), "NOFILTER"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures").toURL(), "PICTURE"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=videos").toURL(), "VIDEO"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=albums").toURL(), "ALBUM"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=pictures").toURL(), "PICTURE"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=videos").toURL(), "VIDEO"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?sort=top&filter=albums").toURL(), "ALBUM"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=pictures&sort=top").toURL(), "PICTURE"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=videos&sort=top").toURL(), "VIDEO"); + testURLs.put(new URI("https://scrolller.com/r/CatsStandingUp?filter=albums&sort=top").toURL(), "ALBUM"); + for (URL url : testURLs.keySet()) { + ScrolllerRipper ripper = new ScrolllerRipper(url); + ripper.setup(); + Assertions.assertEquals(testURLs.get(url), ripper.convertFilterString(ripper.getParameter(ripper.getURL(),"filter"))); + deleteDir(ripper.getWorkingDir()); + } + } + + + +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java index c6ab6a79..20e14442 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VscoRipperTest.java @@ -1,52 +1,52 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import com.rarchives.ripme.ripper.rippers.VscoRipper; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; - -public class VscoRipperTest extends RippersTest { - - /** - * Testing single image. - * - * @throws IOException - */ - @Test - public void testSingleImageRip() throws IOException, URISyntaxException { - VscoRipper ripper = new VscoRipper(new URI("https://vsco.co/jolly-roger/media/597ce449846079297b3f7cf3").toURL()); - testRipper(ripper); - } - - /** - * Tests profile rip., Prevents Bug #679 from happening again. - * https://github.com/RipMeApp/ripme/issues/679 - * - * @throws IOException - */ - @Test - public void testHyphenatedRip() throws IOException, URISyntaxException { - VscoRipper ripper = new VscoRipper(new URI("https://vsco.co/jolly-roger/gallery").toURL()); - testRipper(ripper); - } - - /** - * Make sure it names the folder something sensible. - * - * @throws IOException - */ - @Test - public void testGetGID() throws IOException, URISyntaxException { - URL url = new URI("https://vsco.co/jolly-roger/media/590359c4ade3041f2658f407").toURL(); - - VscoRipper ripper = new VscoRipper(url); - - Assertions.assertEquals("jolly-roger/59035", ripper.getGID(url), "Failed to get GID"); - } - -} +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.VscoRipper; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; + +public class VscoRipperTest extends RippersTest { + + /** + * Testing single image. + * + * @throws IOException + */ + @Test + public void testSingleImageRip() throws IOException, URISyntaxException { + VscoRipper ripper = new VscoRipper(new URI("https://vsco.co/jolly-roger/media/597ce449846079297b3f7cf3").toURL()); + testRipper(ripper); + } + + /** + * Tests profile rip., Prevents Bug #679 from happening again. + * https://github.com/RipMeApp/ripme/issues/679 + * + * @throws IOException + */ + @Test + public void testHyphenatedRip() throws IOException, URISyntaxException { + VscoRipper ripper = new VscoRipper(new URI("https://vsco.co/jolly-roger/gallery").toURL()); + testRipper(ripper); + } + + /** + * Make sure it names the folder something sensible. + * + * @throws IOException + */ + @Test + public void testGetGID() throws IOException, URISyntaxException { + URL url = new URI("https://vsco.co/jolly-roger/media/590359c4ade3041f2658f407").toURL(); + + VscoRipper ripper = new VscoRipper(url); + + Assertions.assertEquals("jolly-roger/59035", ripper.getGID(url), "Failed to get GID"); + } + +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XlecxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XlecxRipperTest.java index 2b943c7d..78eb5a3a 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XlecxRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XlecxRipperTest.java @@ -1,19 +1,19 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - -import com.rarchives.ripme.ripper.rippers.XlecxRipper; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -public class XlecxRipperTest extends RippersTest { - @Test - @Disabled("Broken ripper") - public void testAlbum() throws IOException, URISyntaxException { - XlecxRipper ripper = new XlecxRipper(new URI("http://xlecx.com/4274-black-canary-ravished-prey.html").toURL()); - testRipper(ripper); - } -} +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +import com.rarchives.ripme.ripper.rippers.XlecxRipper; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +public class XlecxRipperTest extends RippersTest { + @Test + @Disabled("Broken ripper") + public void testAlbum() throws IOException, URISyntaxException { + XlecxRipper ripper = new XlecxRipper(new URI("http://xlecx.com/4274-black-canary-ravished-prey.html").toURL()); + testRipper(ripper); + } +} diff --git a/utils/style.sh b/utils/style.sh deleted file mode 100644 index 45bb40e9..00000000 --- a/utils/style.sh +++ /dev/null @@ -1,27 +0,0 @@ -echo "" -echo "=====================================================" -echo "Tabs are not allowed" -echo "-----------------------------------------------------" -git grep -n -P "\t" -- :/*.java | sed -e "s/\t/\x1b[7m--->\x1b[m/g" -echo "=====================================================" - -echo "" -echo "=====================================================" -echo "Trailing whitespace is not allowed" -echo "-----------------------------------------------------" -git grep -n -P "[ \t]+$" -- :/*.java | sed -e "s/\t/\x1b[7m--->\x1b[m/g" | sed -e "s/ /\x1b[7m.\x1b[m/g" | sed -e "s/$/\x1b[7m$\x1b[m/g" -echo "=====================================================" - -echo "" -echo "=====================================================" -echo "'){' is not allowed. Place a space between ')' and '{', i.e. 'if (a) {'" -echo "-----------------------------------------------------" -git grep -n -P "\)\{" -- :/*.java -echo "=====================================================" - -echo "" -echo "=====================================================" -echo "A space is required after keywords (if|else|for|while|do|try|catch|finally)" -echo "-----------------------------------------------------" -git grep -n -P "(\b(if|for|while|catch)\b[(])|(\b(else|do|try|finally)\b[{])" -- :/*.java | sed -r -e "s/(\b(if|for|while|catch)\b[(])|(\b(else|do|try|finally)\b[{])/\x1b[7m\0\x1b[m/g" -echo "=====================================================" diff --git a/utils/stylefix.sh b/utils/stylefix.sh deleted file mode 100644 index dbfad1e1..00000000 --- a/utils/stylefix.sh +++ /dev/null @@ -1,17 +0,0 @@ -echo "" -echo "=====================================================" -echo "Tabs are not allowed (please manually fix tabs)" -echo "-----------------------------------------------------" -git grep -n -P "\t" -- :/*.java | sed -e "s/\t/\x1b[7m--->\x1b[m/g" -echo "=====================================================" - -echo "Removing trailing whitespace..." -git grep -l -P "[ \t]+$" -- :/*.java | xargs -I % sed -i -r -e "s/[ \t]+$//g" % - -echo "Replacing '){' with ') {'..." -git grep -l -P "\)\{" -- :/*.java | xargs -I % sed -i -r -e "s/\)\{/) {/g" % - -echo "Adding space between keywords and punctuation..." -git grep -l -P "(\b(if|for|while|catch)\b[(])" -- :/*.java | xargs -I % sed -i -r -e "s/(\b(if|for|while|catch)\b[(])/\2 (/g" % -git grep -l -P "(\b(else|do|try|finally)\b[{])" -- :/*.java | xargs -I % sed -i -r -e "s/(\b(else|do|try|finally)\b[{])/\2 {/g" % - From eee8f1bd508de5a705951655d9423fd8cddd97f7 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 30 Jul 2023 04:40:46 +0200 Subject: [PATCH 391/512] update to gradle-8.2.1 --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index fae08049..84a0b92f 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From ecf427cdee51ea3bae4b5da42e368a5d960ccab1 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 30 Jul 2023 04:45:14 +0200 Subject: [PATCH 392/512] java-17 is default --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index ecaf37c0..8029ec54 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,7 +1,7 @@ // permits to start the build setting the javac release parameter, no parameter means build for java8: // gradle clean build -PjavacRelease=8 // gradle clean build -PjavacRelease=17 -val javacRelease = (project.findProperty("javacRelease") ?: "11") as String +val javacRelease = (project.findProperty("javacRelease") ?: "17") as String plugins { id("fr.brouillard.oss.gradle.jgitver") version "0.9.1" From 836a74940e094bd0acf2df243ec3225c7c68bed7 Mon Sep 17 00:00:00 2001 From: brantspar <125783125+brantspar@users.noreply.github.com> Date: Sat, 15 Jul 2023 15:14:23 +1000 Subject: [PATCH 393/512] * fixed imagefap ripper (switching from img/src to img/data-src) --- .../ripme/ripper/AbstractHTMLRipper.java | 2 +- .../ripme/ripper/rippers/ImagefapRipper.java | 30 +++++++++++++++++-- .../ripper/rippers/ImagefapRipperTest.java | 12 +++----- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 6a944d8e..f087f980 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -160,7 +160,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { for (String imageURL : imageURLs) { index += 1; - LOGGER.debug("Found image url #" + index + ": " + imageURL); + LOGGER.debug("Found image url #" + index + ": '" + imageURL + "'"); downloadURL(new URL(imageURL), index); if (isStopped() || isThisATest()) { break; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java index 228ae65a..5ef89fbd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java @@ -1,8 +1,11 @@ package com.rarchives.ripme.ripper.rippers; +import java.io.File; +import java.io.FileWriter; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; @@ -10,6 +13,7 @@ import java.util.regex.Pattern; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; @@ -141,6 +145,8 @@ public class ImagefapRipper extends AbstractHTMLRipper { if(image == null) throw new RuntimeException("Unable to extract image URL from single image page! Unable to continue"); + LOGGER.debug("Adding imageURL: '" + image + "'"); + imageURLs.add(image); if (isThisATest()) { break; @@ -177,9 +183,29 @@ public class ImagefapRipper extends AbstractHTMLRipper { sleep(IMAGE_SLEEP_TIME); Document doc = getPageWithRetries(new URL(pageURL)); - return doc.select("img#mainPhoto").attr("src"); + + String framedPhotoUrl = doc.select("img#mainPhoto").attr("data-src"); + + // we use a no query param version of the URL to reduce failure rate because of some query params that change between the li elements and the mainPhotoURL + String noQueryPhotoUrl = framedPhotoUrl.split("\\?")[0]; + + LOGGER.debug("noQueryPhotoUrl: " + noQueryPhotoUrl); + + // we look for a li > a element who's framed attribute starts with the noQueryPhotoUrl (only reference in the page to the full URL) + Elements selectedItem = doc.select("ul.thumbs > li > a[framed^='"+noQueryPhotoUrl+"']"); + + // the fullsize URL is in the href attribute + String fullSizedUrl = selectedItem.attr("href"); + + if("".equals(fullSizedUrl)) + throw new IOException("JSoup full URL extraction failed from '" + selectedItem.html() + "'"); + + LOGGER.debug("fullSizedUrl: " + fullSizedUrl); + + return fullSizedUrl; + } catch (IOException e) { - LOGGER.debug("Unable to get full size image URL from page URL " + pageURL + " because: " + e.getMessage()); + LOGGER.debug("Unable to get full size image URL from page: " + pageURL + " because: " + e.getMessage()); return null; } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java index 765eb4ad..2af7d499 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java @@ -19,12 +19,8 @@ public class ImagefapRipperTest extends RippersTest { Map testURLs = new HashMap<>(); // Album with specific title - testURLs.put(new URI("http://www.imagefap.com/pictures/4649440/Frozen-%28Elsa-and-Anna%29?view=2").toURL(), - "Frozen (Elsa and Anna)"); - - // New URL format - testURLs.put(new URI("http://www.imagefap.com/gallery.php?pgid=fffd68f659befa5535cf78f014e348f1").toURL(), - "imagefap_fffd68f659befa5535cf78f014e348f1"); + testURLs.put(new URI("https://www.imagefap.com/pictures/11365460/Cartoons").toURL(), + "Cartoons"); for (URL url : testURLs.keySet()) { ImagefapRipper ripper = new ImagefapRipper(url); @@ -34,8 +30,8 @@ public class ImagefapRipperTest extends RippersTest { @Test @Tag("flaky") public void testImagefapGetAlbumTitle() throws IOException, URISyntaxException { - URL url = new URI("https://www.imagefap.com/gallery.php?gid=7789753").toURL(); + URL url = new URI("https://www.imagefap.com/pictures/11365460/Cartoons").toURL(); ImagefapRipper ripper = new ImagefapRipper(url); - Assertions.assertEquals("imagefap_Red.Heels.Lover.In.Love_7789753", ripper.getAlbumTitle(url)); + Assertions.assertEquals("imagefap_Cartoons_11365460", ripper.getAlbumTitle(url)); } } From 4531976e15213acade798932cb3f69cc791cd556 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 30 Jul 2023 05:33:31 +0200 Subject: [PATCH 394/512] release 2.1.4 releasing again 2.1.4 as github download link includes the tag name, and the java code uses the git hashed tag to strip out the semantic version. download link: https://github.com/ripmeapp2/ripme/releases/download/2.1.4/ripme-2.1.4-38-836a7494.jar which the code calculates. --- ripme.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ripme.json b/ripme.json index eef798d0..4efea259 100644 --- a/ripme.json +++ b/ripme.json @@ -1,7 +1,8 @@ { - "latestVersion": "2.1.3-15-1b83dc68", - "currentHash": "1b83dc68aea66bce3627a05b2fa2ab568044e847", + "latestVersion": "2.1.4-38-836a7494", + "currentHash": "a82ad80fe406a01adf37b3de403245ad78e0669aa96b91d6befe29aaf88200d2", "changeList": [ + "2.1.4-38-836a7494: fixed imagefap ripper.", "2.1.3-15-1b83dc68: relative path now from working dir to subfolder, allowing images to be put in subfolder with same filename, sanatize reddit titles saved as files, additional logging in AbstractHTMLRipper.", "2.1.2-23-e5438e85: caching of first page, retry sleep time, nhentai fixed", "2.1.2-3-ea90b172: better sanitize filenames for windows, save config on update value. reddit, print exceptions in loops and continue.", From 037655275538d7b79a2ae8d1795e3739f7c7dbb2 Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 27 Sep 2023 19:16:46 +0200 Subject: [PATCH 395/512] gradle-8.3 upgrade --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 84a0b92f..db9a6b82 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From f69943f2f464cc3ec232e0913c01332eb1b4ead5 Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 27 Sep 2023 23:32:20 +0200 Subject: [PATCH 396/512] require java-17 --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9887f724..f6f0868d 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,8 @@ RipMe is maintained with ♥️ and in our limited free time by **[@MetaPrime](h # About -RipMe is an album ripper for various websites. It is a cross-platform tool that runs on your computer, and requires Java 11. RipMe has been tested and confirmed working on Windows, Linux and MacOS. +RipMe is an album ripper for various websites. It is a cross-platform tool that runs on your computer, and +requires Java 17. RipMe has been tested and confirmed working on Windows, Linux and MacOS. ![Screenshot](https://i.imgur.com/UCQNjeg.png) From ef884f044bd0334c7de98a7c0e1290d716c57533 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 29 Sep 2023 06:29:39 +0200 Subject: [PATCH 397/512] github build with java-21, upload with java-17. build.sh should not test, people run it sometimes to build. --- .github/workflows/gradle.yml | 8 ++++---- build.bat | 1 - build.sh | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 8c39b1f9..8c2f0c4f 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -15,8 +15,8 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - java: [20] - include: # test newest java on one os only, upload from ubuntu java11 + java: [21] + include: # test old java on one os only, upload from ubuntu java-17 - os: ubuntu-latest java: 17 - os: ubuntu-latest @@ -31,10 +31,10 @@ jobs: uses: FranzDiebold/github-env-vars-action@v2 - name: Set up java - uses: actions/setup-java@v3.0.0 + uses: actions/setup-java@v3.13.0 with: java-version: ${{ matrix.java }} - distribution: temurin + distribution: zulu cache: gradle - name: Build with Gradle diff --git a/build.bat b/build.bat index 719662c9..f6bf32a6 100755 --- a/build.bat +++ b/build.bat @@ -1,2 +1 @@ ./gradlew clean build -x test -./gradlew testAll diff --git a/build.sh b/build.sh index 68578762..d4dbe3b8 100755 --- a/build.sh +++ b/build.sh @@ -1,3 +1,2 @@ #!/usr/bin/env bash ./gradlew clean build -x test -./gradlew testAll From 0ccf2844f89b107dddaf3bcb172bcc04f34f3a17 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 30 Sep 2023 01:54:22 +0200 Subject: [PATCH 398/512] gfycat shut down --- .../ripme/ripper/rippers/GfycatRipper.java | 160 ------------------ .../com/rarchives/ripme/utils/RipUtils.java | 13 -- .../tst/ripper/rippers/GfycatRipperTest.java | 53 ------ 3 files changed, 226 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java delete mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java deleted file mode 100644 index c542c6dc..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java +++ /dev/null @@ -1,160 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import org.json.JSONArray; -import org.json.JSONObject; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import com.rarchives.ripme.utils.Http; - - -public class GfycatRipper extends AbstractHTMLRipper { - - private static final String HOST = "gfycat.com"; - String username = ""; - String cursor = ""; - String count = "30"; - String REFERRER = "www.reddit.com"; - - - - public GfycatRipper(URL url) throws IOException { - super(new URL(url.toExternalForm().split("-")[0].replace("thumbs.", ""))); - } - - @Override - public String getDomain() { - return "gfycat.com"; - } - - @Override - public String getHost() { - return "gfycat"; - } - - @Override - public boolean canRip(URL url) { - return url.getHost().endsWith(HOST); - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - String sUrl = url.toExternalForm(); - sUrl = sUrl.replace("/gifs/detail", ""); - sUrl = sUrl.replace("/amp", ""); - return new URL(sUrl); - } - - public boolean isProfile() { - Pattern p = Pattern.compile("^https?://[wm.]*gfycat\\.com/@([a-zA-Z0-9\\.\\-\\_]+).*$"); - Matcher m = p.matcher(url.toExternalForm()); - return m.matches(); - } - - @Override - public Document getFirstPage() throws IOException { - if (!isProfile()) { - return Http.url(url).referrer(REFERRER).get(); - } else { - username = getGID(url); - return Http.url(new URL("https://api.gfycat.com/v1/users/" + username + "/gfycats")).referrer((REFERRER)).ignoreContentType().get(); - } - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://(?:thumbs\\.|[wm\\.]*)gfycat\\.com/@?([a-zA-Z0-9\\.\\-\\_]+).*$"); - Matcher m = p.matcher(url.toExternalForm()); - - if (m.matches()) - return m.group(1); - - throw new MalformedURLException( - "Expected gfycat.com format: " - + "gfycat.com/id or " - + "thumbs.gfycat.com/id.gif" - + " Got: " + url); - } - - private String stripHTMLTags(String t) { - t = t.replaceAll("\n" + - " \n" + - " ", ""); - t = t.replaceAll("\n" + - "", ""); - t = t.replaceAll("\n", ""); - t = t.replaceAll("=\"\"", ""); - return t; - } - - @Override - public Document getNextPage(Document doc) throws IOException { - if (cursor.equals("")) { - throw new IOException("No more pages"); - } - return Http.url(new URL("https://api.gfycat.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor)).ignoreContentType().get(); - } - - @Override - public List getURLsFromPage(Document doc) { - List result = new ArrayList<>(); - if (isProfile()) { - JSONObject page = new JSONObject(stripHTMLTags(doc.html())); - JSONArray content = page.getJSONArray("gfycats"); - for (int i = 0; i < content.length(); i++) { - result.add(content.getJSONObject(i).getString("mp4Url")); - } - cursor = page.getString("cursor"); - } else { - Elements videos = doc.select("script"); - for (Element el : videos) { - String json = el.html(); - if (json.startsWith("{")) { - JSONObject page = new JSONObject(json); - result.add(page.getJSONObject("video").getString("contentUrl")); - } - } - } - return result; - } - - /** - * Helper method for retrieving video URLs. - * @param url URL to gfycat page - * @return URL to video - * @throws IOException - */ - public static String getVideoURL(URL url) throws IOException { - LOGGER.info("Retrieving " + url.toExternalForm()); - - //Sanitize the URL first - url = new URL(url.toExternalForm().replace("/gifs/detail", "")); - - Document doc = Http.url(url).get(); - Elements videos = doc.select("script"); - for (Element el : videos) { - String json = el.html(); - if (json.startsWith("{")) { - JSONObject page = new JSONObject(json); - return page.getJSONObject("video").getString("contentUrl"); - } - } - throw new IOException(); - } -} diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 14aa6dce..85d092e8 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -15,7 +15,6 @@ import com.rarchives.ripme.ripper.rippers.EromeRipper; import com.rarchives.ripme.ripper.rippers.ImgurRipper; import com.rarchives.ripme.ripper.rippers.RedgifsRipper; import com.rarchives.ripme.ripper.rippers.VidbleRipper; -import com.rarchives.ripme.ripper.rippers.GfycatRipper; import com.rarchives.ripme.ripper.rippers.SoundgasmRipper; import org.apache.commons.lang.math.NumberUtils; import org.apache.logging.log4j.LogManager; @@ -69,18 +68,6 @@ public class RipUtils { return result; } - else if (url.getHost().endsWith("gfycat.com")) { - try { - logger.debug("Fetching gfycat page " + url); - String videoURL = GfycatRipper.getVideoURL(url); - logger.debug("Got gfycat URL: " + videoURL); - result.add(new URI(videoURL).toURL()); - } catch (IOException | URISyntaxException e) { - // Do nothing - logger.warn("Exception while retrieving gfycat page:", e); - } - return result; - } else if (url.getHost().endsWith("redgifs.com") || url.getHost().endsWith("gifdeliverynetwork.com")) { try { logger.debug("Fetching redgifs page " + url); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java deleted file mode 100644 index c8400d56..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import com.rarchives.ripme.ripper.rippers.GfycatRipper; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - - -public class GfycatRipperTest extends RippersTest { - - /** - * Rips correctly formatted URL directly from Gfycat - */ - @Test - public void testGfycatGoodURL() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/TemptingExcellentIchthyosaurs").toURL()); - testRipper(ripper); - } - /** - * Rips badly formatted URL directly from Gfycat - */ - public void testGfycatBadURL() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/gifs/detail/limitedtestyamericancrow").toURL()); - testRipper(ripper); - } - - /** - * Rips a Gfycat profile - */ - public void testGfycatProfile() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/@golbanstorage").toURL()); - testRipper(ripper); - } - - /** - * Rips a Gfycat amp link - * @throws IOException - */ - public void testGfycatAmp() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/amp/TemptingExcellentIchthyosaurs").toURL()); - testRipper(ripper); - } - - /** - * Rips a Gfycat profile with special characters in username - */ - public void testGfycatSpecialChar() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/@rsss.kr").toURL()); - testRipper(ripper); - } -} From 45e6c4a6163fba09255d8e2ec52692217c0d3c88 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 30 Sep 2023 10:10:45 +0200 Subject: [PATCH 399/512] upload java-17 build --- .github/workflows/gradle.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 8c2f0c4f..0a1e892a 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -19,7 +19,6 @@ jobs: include: # test old java on one os only, upload from ubuntu java-17 - os: ubuntu-latest java: 17 - - os: ubuntu-latest upload: true steps: From 9c2404ec26e1b75cc01d6a109429e6ac76c79fd3 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 30 Sep 2023 16:47:11 +0200 Subject: [PATCH 400/512] gradle-8.3 deprecations fixed --- build.gradle.kts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 8029ec54..aee5440f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -29,8 +29,9 @@ dependencies { implementation("org.apache.logging.log4j:log4j-api:2.20.0") implementation("org.apache.logging.log4j:log4j-core:2.20.0") implementation("org.graalvm.js:js:22.3.2") - testImplementation(enforcedPlatform("org.junit:junit-bom:5.9.3")) + testImplementation(enforcedPlatform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") } group = "com.rarchives.ripme" @@ -128,7 +129,7 @@ tasks.jacocoTestReport { reports { xml.required.set(false) csv.required.set(false) - html.outputLocation.set(file("${buildDir}/jacocoHtml")) + html.outputLocation.set(file("${layout.buildDirectory}/jacocoHtml")) } } From ba51d7b5697cf922f73415a95954e93db0b0db0d Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 30 Sep 2023 17:28:46 +0200 Subject: [PATCH 401/512] list and address deprecations in java --- build.gradle.kts | 2 ++ .../java/com/rarchives/ripme/ui/ContextMenuMouseListener.java | 2 +- .../java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java | 2 +- .../java/com/rarchives/ripme/ui/QueueMenuMouseListener.java | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index aee5440f..e57b8579 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -79,6 +79,8 @@ publishing { tasks.withType { options.encoding = "UTF-8" + val compilerArgs = options.compilerArgs + compilerArgs.addAll(listOf("-Xlint:deprecation")) } tasks.test { diff --git a/src/main/java/com/rarchives/ripme/ui/ContextMenuMouseListener.java b/src/main/java/com/rarchives/ripme/ui/ContextMenuMouseListener.java index 94348411..ad75ede5 100644 --- a/src/main/java/com/rarchives/ripme/ui/ContextMenuMouseListener.java +++ b/src/main/java/com/rarchives/ripme/ui/ContextMenuMouseListener.java @@ -98,7 +98,7 @@ public class ContextMenuMouseListener extends MouseAdapter { @Override public void mouseClicked(MouseEvent e) { - if (e.getModifiers() == InputEvent.BUTTON3_MASK) { + if (e.getModifiersEx() == InputEvent.BUTTON3_DOWN_MASK) { if (!(e.getSource() instanceof JTextComponent)) { return; } diff --git a/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java b/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java index 9044531f..cf288f2d 100644 --- a/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java +++ b/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java @@ -63,7 +63,7 @@ class HistoryMenuMouseListener extends MouseAdapter { @Override public void mouseClicked(MouseEvent e) { - if (e.getModifiers() == InputEvent.BUTTON3_MASK) { + if (e.getModifiersEx() == InputEvent.BUTTON3_DOWN_MASK) { if (!(e.getSource() instanceof JTable)) { return; } diff --git a/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java b/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java index 19911ee2..08adce80 100644 --- a/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java +++ b/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java @@ -61,7 +61,7 @@ class QueueMenuMouseListener extends MouseAdapter { @SuppressWarnings("unchecked") @Override public void mouseClicked(MouseEvent e) { - if (e.getModifiers() == InputEvent.BUTTON3_MASK) { + if (e.getModifiersEx() == InputEvent.BUTTON3_DOWN_MASK) { if (!(e.getSource() instanceof JList)) { return; } From 52e7fbb767483e937d1840450de4379b3012f0e5 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 30 Sep 2023 19:50:04 +0200 Subject: [PATCH 402/512] release 2.1.5-8-ba51d7b, runningn with java-17 --- ripme.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ripme.json b/ripme.json index 4efea259..7a513418 100644 --- a/ripme.json +++ b/ripme.json @@ -1,7 +1,8 @@ { - "latestVersion": "2.1.4-38-836a7494", - "currentHash": "a82ad80fe406a01adf37b3de403245ad78e0669aa96b91d6befe29aaf88200d2", + "latestVersion": "2.1.5-8-ba51d7b", + "currentHash": "307406fdde89a7fbd7817ea9d0874728706d909f0507d94bdd18574cad193acf", "changeList": [ + "2.1.5-8-ba51d7b: ripme running with java-17", "2.1.4-38-836a7494: fixed imagefap ripper.", "2.1.3-15-1b83dc68: relative path now from working dir to subfolder, allowing images to be put in subfolder with same filename, sanatize reddit titles saved as files, additional logging in AbstractHTMLRipper.", "2.1.2-23-e5438e85: caching of first page, retry sleep time, nhentai fixed", From 68189f27aa5131b711d92777dbe09c73e39878f4 Mon Sep 17 00:00:00 2001 From: pesho1323 <147543994+pesho1323@users.noreply.github.com> Date: Tue, 10 Oct 2023 21:25:26 +0300 Subject: [PATCH 403/512] Erome ripper does not download images fixed --- .../com/rarchives/ripme/ripper/rippers/EromeRipper.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java index eb5ad8ea..bc1e9df9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java @@ -126,17 +126,16 @@ public class EromeRipper extends AbstractHTMLRipper { private List getMediaFromPage(Document doc) { List results = new ArrayList<>(); for (Element el : doc.select("img.img-front")) { - if (el.hasAttr("src")) { + if (el.hasAttr("data-src")) { + //to add images that are not loaded( as all images are lasyloaded as we scroll). + results.add(el.attr("data-src")); + } else if (el.hasAttr("src")) { if (el.attr("src").startsWith("https:")) { results.add(el.attr("src")); } else { results.add("https:" + el.attr("src")); } - } else if (el.hasAttr("data-src")) { - //to add images that are not loaded( as all images are lasyloaded as we scroll). - results.add(el.attr("data-src")); } - } for (Element el : doc.select("source[label=HD]")) { if (el.attr("src").startsWith("https:")) { From aeeac429fe1c7b55368649b7376ad55a4bd64433 Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 12 Oct 2023 08:27:50 +0200 Subject: [PATCH 404/512] release 2.1.6 --- ripme.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ripme.json b/ripme.json index 7a513418..9d23c24a 100644 --- a/ripme.json +++ b/ripme.json @@ -1,8 +1,9 @@ { - "latestVersion": "2.1.5-8-ba51d7b", - "currentHash": "307406fdde89a7fbd7817ea9d0874728706d909f0507d94bdd18574cad193acf", + "latestVersion": "2.1.6-1-68189f27", + "currentHash": "adffec078cc6e1da42699f874176744622a00a1dace000f2ef0c7dad28957faa", "changeList": [ - "2.1.5-8-ba51d7b: ripme running with java-17", + "2.1.6-1-68189f27: erome fix.", + "2.1.5-8-ba51d7b: ripme running with java-17.", "2.1.4-38-836a7494: fixed imagefap ripper.", "2.1.3-15-1b83dc68: relative path now from working dir to subfolder, allowing images to be put in subfolder with same filename, sanatize reddit titles saved as files, additional logging in AbstractHTMLRipper.", "2.1.2-23-e5438e85: caching of first page, retry sleep time, nhentai fixed", From 0bb8d8b2d98f70c712403937f3618f5bbe5bdd1d Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 14 Oct 2023 14:10:46 +0200 Subject: [PATCH 405/512] permit to override the jgitver version for packaging a tar ball, jgitver takes the default version of 0.0.0, not practical. putting a version into a file would be possible, but we try to produce a new version with every commit, which then can be released if tests pass, without rebuild. this means the source code should NOT have the version in the commit, but derived. for tar balls, we know the name of the tar, so its easy to set the same version in a script, or put a desired version. pay attention, that ripme checks the version towards github, and suggests an update, if the running version is less than the released one. fixes #145. --- build.gradle.kts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index e57b8579..a5c77e48 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,6 +1,9 @@ -// permits to start the build setting the javac release parameter, no parameter means build for java8: -// gradle clean build -PjavacRelease=8 -// gradle clean build -PjavacRelease=17 +// the build derives a version with the jgitver plugin out of a tag in the git history. when there is no +// git repo, the jgitver default would be 0.0.0. one can override this version with a parameter. also, permit +// to start the build setting the javac release parameter, no parameter means build for java-17: +// gradle clean build -PjavacRelease=21 +// gradle clean build -PcustomVersion=1.0.0-10-asdf +val customVersion = (project.findProperty("customVersion") ?: "") as String val javacRelease = (project.findProperty("javacRelease") ?: "17") as String plugins { @@ -48,6 +51,12 @@ jgitver { useGitCommitID = true } +afterEvaluate { + if (customVersion != "") { + project.version = customVersion + } +} + tasks.compileJava { options.release.set(Integer.parseInt(javacRelease)) } From c329343515e558d7b155e846c7a50edec3c38f10 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 14 Oct 2023 14:55:21 +0200 Subject: [PATCH 406/512] twitter, do not exclude replies, fixes #41 --- .../java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java index 1a1bf1ab..2ce65834 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TwitterRipper.java @@ -121,7 +121,7 @@ public class TwitterRipper extends AbstractJSONRipper { case ACCOUNT: req.append("https://api.twitter.com/1.1/statuses/user_timeline.json") .append("?screen_name=" + this.accountName).append("&include_entities=true") - .append("&exclude_replies=true").append("&trim_user=true").append("&count=" + MAX_ITEMS_REQUEST) + .append("&exclude_replies=false").append("&trim_user=true").append("&count=" + MAX_ITEMS_REQUEST) .append("&tweet_mode=extended"); break; case SEARCH:// Only get tweets from last week From 3fcad4f8cb71d942c8de258e416c07ed7f1f28da Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 14 Oct 2023 15:20:48 +0200 Subject: [PATCH 407/512] adust hitomi regex, see #43 --- .../java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java index 3196c139..8d02ff56 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java @@ -35,14 +35,14 @@ public class HitomiRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("https://hitomi.la/galleries/([\\d]+).html"); + Pattern p = Pattern.compile("https://hitomi.la/(cg|doujinshi|gamecg|manga)/(.+).html"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { galleryId = m.group(1); return m.group(1); } throw new MalformedURLException("Expected hitomi URL format: " + - "https://hitomi.la/galleries/ID.html - got " + url + " instead"); + "https://hitomi.la/(cg|doujinshi|gamecg|manga)/ID.html - got " + url + " instead"); } @Override From 0b261dd5d7ee6df1ccfd98bc3d5808c8ab8ec5f3 Mon Sep 17 00:00:00 2001 From: pesho1323 <147543994+pesho1323@users.noreply.github.com> Date: Tue, 17 Oct 2023 18:19:39 +0300 Subject: [PATCH 408/512] Add Multporn ripper --- .../ripme/ripper/rippers/MultpornRipper.java | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java new file mode 100644 index 00000000..b4a0597f --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java @@ -0,0 +1,71 @@ +package com.rarchives.ripme.ripper.rippers; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class MultpornRipper extends AbstractHTMLRipper { + + public MultpornRipper(URL url) throws IOException { + super(url); + } + + @Override + protected String getDomain() { + return "multporn.net"; + } + + @Override + public String getHost() { + return "multporn"; + } + + @Override + public String getGID(URL url) throws MalformedURLException, URISyntaxException { + Pattern p = Pattern.compile("^https?://multporn\\.net/node/(\\d+).*$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + + try { + String nodeHref = Http.url(url).get().select(".simple-mode-switcher").attr("href"); + System.out.println(nodeHref); + p = Pattern.compile("/node/(\\d+)/.*"); + m = p.matcher(nodeHref); + if (m.matches()) { + this.url = new URL("https://multporn.net" + nodeHref); + return m.group(1); + } + }catch (Exception ignored){}; + + throw new MalformedURLException("Expected multporn.net URL format: " + + "multporn.net/comics/comicid / multporn.net/node/id/* - got " + url + " instead"); + } + + @Override + protected List getURLsFromPage(Document page) { + List imageURLs = new ArrayList<>(); + Elements thumbs = page.select(".mfp-gallery-image .mfp-item"); + for (Element el : thumbs) { + imageURLs.add(el.attr("href")); + } + return imageURLs; + } + + @Override + protected void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), null); + } +} \ No newline at end of file From 2843d4e91404dc6c904b0b6723830e2f3d357ea5 Mon Sep 17 00:00:00 2001 From: pesho1323 <147543994+pesho1323@users.noreply.github.com> Date: Tue, 17 Oct 2023 18:26:31 +0300 Subject: [PATCH 409/512] small pattern change --- .../com/rarchives/ripme/ripper/rippers/MultpornRipper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java index b4a0597f..06bcfdbb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java @@ -33,7 +33,7 @@ public class MultpornRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException, URISyntaxException { - Pattern p = Pattern.compile("^https?://multporn\\.net/node/(\\d+).*$"); + Pattern p = Pattern.compile("^https?://multporn\\.net/node/(\\d+)/.*$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); @@ -68,4 +68,4 @@ public class MultpornRipper extends AbstractHTMLRipper { protected void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), null); } -} \ No newline at end of file +} From 0cca139e8e6119584b818e88e133d728a2901bed Mon Sep 17 00:00:00 2001 From: pesho1323 <147543994+pesho1323@users.noreply.github.com> Date: Tue, 17 Oct 2023 19:45:53 +0300 Subject: [PATCH 410/512] Remove print --- .../java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java index 06bcfdbb..42683694 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java @@ -41,7 +41,6 @@ public class MultpornRipper extends AbstractHTMLRipper { try { String nodeHref = Http.url(url).get().select(".simple-mode-switcher").attr("href"); - System.out.println(nodeHref); p = Pattern.compile("/node/(\\d+)/.*"); m = p.matcher(nodeHref); if (m.matches()) { From 74fe3a7356a1477856f19630db5337c0068d528e Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 27 Oct 2023 21:53:57 +0200 Subject: [PATCH 411/512] Locale.forLanguageTag instead of deprecated 'new Locale' --- src/main/java/com/rarchives/ripme/utils/Utils.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index e24677ef..a1f85d11 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -735,15 +735,14 @@ public class Utils { public static ResourceBundle getResourceBundle(String langSelect) { if (langSelect == null) { if (!getConfigString("lang", "").equals("")) { - String[] langCode = getConfigString("lang", "").split("_"); LOGGER.info("Setting locale to " + getConfigString("lang", "")); - return ResourceBundle.getBundle("LabelsBundle", new Locale(langCode[0], langCode[1]), + return ResourceBundle.getBundle("LabelsBundle", Locale.forLanguageTag(getConfigString("lang", "")), new UTF8Control()); } } else { String[] langCode = langSelect.split("_"); LOGGER.info("Setting locale to " + langSelect); - return ResourceBundle.getBundle("LabelsBundle", new Locale(langCode[0], langCode[1]), new UTF8Control()); + return ResourceBundle.getBundle("LabelsBundle", Locale.forLanguageTag(langSelect), new UTF8Control()); } try { LOGGER.info("Setting locale to default"); From dd6ac9bc125dd955a650355da32e2cb34fb515df Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 27 Oct 2023 21:59:28 +0200 Subject: [PATCH 412/512] new URL deprecated, use URI --- .../ripme/ripper/AbstractHTMLRipper.java | 3 +- .../ripme/ripper/DownloadFileThread.java | 9 ++-- .../ripper/rippers/EightmusesRipper.java | 6 ++- .../ripme/ripper/rippers/EroShareRipper.java | 8 ++-- .../ripme/ripper/rippers/ErofusRipper.java | 6 ++- .../ripme/ripper/rippers/ImagefapRipper.java | 14 +++--- .../ripper/rippers/JagodibujaRipper.java | 6 ++- .../ripme/ripper/rippers/LusciousRipper.java | 7 +-- .../ripme/ripper/rippers/MultpornRipper.java | 3 +- .../ripme/ripper/rippers/RedgifsRipper.java | 46 +++++++++++-------- .../ripme/ripper/rippers/TapasticRipper.java | 6 ++- .../ripme/ripper/rippers/TsuminoRipper.java | 6 ++- .../ripme/ripper/rippers/VidbleRipper.java | 6 ++- .../com/rarchives/ripme/utils/RipUtils.java | 4 +- 14 files changed, 76 insertions(+), 54 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index f087f980..6380a1c6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -3,6 +3,7 @@ package com.rarchives.ripme.ripper; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.*; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -54,7 +55,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { public Document getNextPage(Document doc) throws IOException, URISyntaxException { return null; } - protected abstract List getURLsFromPage(Document page); + protected abstract List getURLsFromPage(Document page) throws UnsupportedEncodingException; protected List getDescriptionsFromPage(Document doc) throws IOException { throw new IOException("getDescriptionsFromPage not implemented"); // Do I do this or make an abstract function? } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index a6722971..e9c6f242 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -1,10 +1,7 @@ package com.rarchives.ripme.ripper; import java.io.*; -import java.net.HttpURLConnection; -import java.net.SocketTimeoutException; -import java.net.URL; -import java.net.URLConnection; +import java.net.*; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Arrays; @@ -152,7 +149,7 @@ class DownloadFileThread implements Runnable { redirected = true; } String location = huc.getHeaderField("Location"); - urlToDownload = new URL(location); + urlToDownload = new URI(location).toURL(); // Throw exception so download can be retried throw new IOException("Redirect status code " + statusCode + " - redirect to " + location); } @@ -284,7 +281,7 @@ class DownloadFileThread implements Runnable { "HTTP status code " + hse.getStatusCode() + " while downloading " + url.toExternalForm()); return; } - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { logger.debug("IOException", e); logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + e.getMessage()); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java index c9984569..f60ced28 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; @@ -114,7 +116,7 @@ public class EightmusesRipper extends AbstractHTMLRipper { try { for (int i = 0; i != json.getJSONArray("pictures").length(); i++) { image = "https://www.8muses.com/image/fl/" + json.getJSONArray("pictures").getJSONObject(i).getString("publicUri"); - URL imageUrl = new URL(image); + URL imageUrl = new URI(image).toURL(); addURLToDownload(imageUrl, getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, getPrefixShort(x), "", null, true); // X is our page index x++; @@ -123,7 +125,7 @@ public class EightmusesRipper extends AbstractHTMLRipper { } } return imageURLs; - } catch (MalformedURLException e) { + } catch (MalformedURLException | URISyntaxException e) { LOGGER.error("\"" + image + "\" is malformed"); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java index bcdca5e4..d7b8015a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java @@ -7,6 +7,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -194,7 +196,7 @@ public class EroShareRipper extends AbstractHTMLRipper { throw new MalformedURLException("eroshare album not found in " + url + ", expected https://eroshare.com/album or eroshae.com/album"); } - public static List getURLs(URL url) throws IOException{ + public static List getURLs(URL url) throws IOException, URISyntaxException { Response resp = Http.url(url) .ignoreContentType() @@ -208,7 +210,7 @@ public class EroShareRipper extends AbstractHTMLRipper { for (Element img : imgs) { if (img.hasClass("album-image")) { String imageURL = img.attr("src"); - URLs.add(new URL(imageURL)); + URLs.add(new URI(imageURL).toURL()); } } //Videos @@ -217,7 +219,7 @@ public class EroShareRipper extends AbstractHTMLRipper { if (vid.hasClass("album-video")) { Elements source = vid.getElementsByTag("source"); String videoURL = source.first().attr("src"); - URLs.add(new URL(videoURL)); + URLs.add(new URI(videoURL).toURL()); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java index cd95d7fc..95528470 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java @@ -9,6 +9,8 @@ import org.jsoup.select.Elements; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; @@ -89,8 +91,8 @@ public class ErofusRipper extends AbstractHTMLRipper { Map opts = new HashMap(); opts.put("subdirectory", page.title().replaceAll(" \\| Erofus - Sex and Porn Comics", "").replaceAll(" ", "_")); opts.put("prefix", getPrefix(x)); - addURLToDownload(new URL(image), opts); - } catch (MalformedURLException e) { + addURLToDownload(new URI(image).toURL(), opts); + } catch (MalformedURLException | URISyntaxException e) { LOGGER.info(e.getMessage()); } x++; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java index 5ef89fbd..bcd5900f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java @@ -4,6 +4,8 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.text.MessageFormat; import java.util.ArrayList; @@ -51,11 +53,11 @@ public class ImagefapRipper extends AbstractHTMLRipper { * Reformat given URL into the desired format (all images on single page) */ @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { String gid = getGID(url); String newURL = "https://www.imagefap.com/pictures/" + gid + "/random-string"; LOGGER.debug("Changed URL from " + url + " to " + newURL); - return new URL(newURL); + return new URI(newURL).toURL(); } @Override @@ -106,7 +108,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { } @Override - public Document getNextPage(Document doc) throws IOException { + public Document getNextPage(Document doc) throws IOException, URISyntaxException { String nextURL = null; for (Element a : doc.select("a.link3")) { if (a.text().contains("next")) { @@ -124,7 +126,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { LOGGER.info("Attempting to load next page URL: " + nextURL); // Load next page - Document nextPage = getPageWithRetries(new URL(nextURL)); + Document nextPage = getPageWithRetries(new URI(nextURL).toURL()); return nextPage; } @@ -182,7 +184,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { // Sleep before fetching image. sleep(IMAGE_SLEEP_TIME); - Document doc = getPageWithRetries(new URL(pageURL)); + Document doc = getPageWithRetries(new URI(pageURL).toURL()); String framedPhotoUrl = doc.select("img#mainPhoto").attr("data-src"); @@ -204,7 +206,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { return fullSizedUrl; - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { LOGGER.debug("Unable to get full size image URL from page: " + pageURL + " because: " + e.getMessage()); return null; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/JagodibujaRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/JagodibujaRipper.java index 2c16b2e5..2f2d5c33 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/JagodibujaRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/JagodibujaRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -56,8 +58,8 @@ public class JagodibujaRipper extends AbstractHTMLRipper { Element elem = comicPage.select("span.full-size-link > a").first(); LOGGER.info("Got link " + elem.attr("href")); try { - addURLToDownload(new URL(elem.attr("href")), ""); - } catch (MalformedURLException e) { + addURLToDownload(new URI(elem.attr("href")).toURL(), ""); + } catch (MalformedURLException | URISyntaxException e) { LOGGER.warn("Malformed URL"); e.printStackTrace(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java index 930eb38d..099eaf73 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java @@ -9,6 +9,7 @@ import org.jsoup.select.Elements; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -95,7 +96,7 @@ public class LusciousRipper extends AbstractHTMLRipper { sanitizedUrl = sanitizedUrl.replaceFirst( "^https?://(?:members\\.|legacy\\.|www\\.)?luscious.net", "https://legacy.luscious.net"); - return new URL(sanitizedUrl); + return new URI(sanitizedUrl).toURL(); } throw new Exception("ERROR: Unable to sanitize url."); @@ -142,9 +143,9 @@ public class LusciousRipper extends AbstractHTMLRipper { } //If a valid download url was found. - addURLToDownload(new URL(downloadUrl), getPrefix(index)); + addURLToDownload(new URI(downloadUrl).toURL(), getPrefix(index)); - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { LOGGER.error("Error downloadiong url " + url, e); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java index 42683694..cdc873f2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MultpornRipper.java @@ -8,6 +8,7 @@ import org.jsoup.select.Elements; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -44,7 +45,7 @@ public class MultpornRipper extends AbstractHTMLRipper { p = Pattern.compile("/node/(\\d+)/.*"); m = p.matcher(nodeHref); if (m.matches()) { - this.url = new URL("https://multporn.net" + nodeHref); + this.url = new URI("https://multporn.net" + nodeHref).toURL(); return m.group(1); } }catch (Exception ignored){}; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java index 3c8547a9..472d6d3a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java @@ -11,6 +11,8 @@ import org.jsoup.select.Elements; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -29,8 +31,8 @@ public class RedgifsRipper extends AbstractHTMLRipper { int searchCount = 150; int searchStart = 0; - public RedgifsRipper(URL url) throws IOException { - super(new URL(url.toExternalForm().replace("thumbs.", ""))); + public RedgifsRipper(URL url) throws IOException, URISyntaxException { + super(new URI(url.toExternalForm().replace("thumbs.", "")).toURL()); } @Override @@ -47,12 +49,12 @@ public class RedgifsRipper extends AbstractHTMLRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { String sUrl = url.toExternalForm(); sUrl = sUrl.replace("/gifs/detail", ""); sUrl = sUrl.replace("/amp", ""); sUrl = sUrl.replace("gifdeliverynetwork.com", "redgifs.com/watch"); - return new URL(sUrl); + return new URI(sUrl).toURL(); } public Matcher isProfile() { @@ -72,16 +74,20 @@ public class RedgifsRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - if (!isProfile().matches() && !isSearch().matches()) { - return Http.url(url).get(); - } else if (isSearch().matches()) { - searchText = getGID(url).replace("-", " "); - return Http.url( - new URL("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart*searchCount)).ignoreContentType().get(); - } else { - username = getGID(url); - return Http.url(new URL("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count)) - .ignoreContentType().get(); + try { + if (!isProfile().matches() && !isSearch().matches()) { + return Http.url(url).get(); + } else if (isSearch().matches()) { + searchText = getGID(url).replace("-", " "); + return Http.url( + new URI("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart * searchCount).toURL()).ignoreContentType().get(); + } else { + username = getGID(url); + return Http.url(new URI("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count).toURL()) + .ignoreContentType().get(); + } + } catch (URISyntaxException e) { + throw new IOException(e); } } @@ -124,18 +130,18 @@ public class RedgifsRipper extends AbstractHTMLRipper { } @Override - public Document getNextPage(Document doc) throws IOException { + public Document getNextPage(Document doc) throws IOException, URISyntaxException { if (isSearch().matches()) { Document d = Http.url( - new URL("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText - + "&count=" + searchCount + "&start=" + searchCount*++searchStart)) + new URI("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + + "&count=" + searchCount + "&start=" + searchCount*++searchStart).toURL()) .ignoreContentType().get(); return (hasURLs(d).isEmpty()) ? null : d; } else { if (cursor.equals("") || cursor.equals("null")) { return null; } else { - Document d = Http.url(new URL("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor)).ignoreContentType().get(); + Document d = Http.url(new URI("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor).toURL()).ignoreContentType().get(); return (hasURLs(d).isEmpty()) ? null : d; } } @@ -182,11 +188,11 @@ public class RedgifsRipper extends AbstractHTMLRipper { * @return URL to video * @throws IOException */ - public static String getVideoURL(URL url) throws IOException { + public static String getVideoURL(URL url) throws IOException, URISyntaxException { LOGGER.info("Retrieving " + url.toExternalForm()); //Sanitize the URL first - url = new URL(url.toExternalForm().replace("/gifs/detail", "")); + url = new URI(url.toExternalForm().replace("/gifs/detail", "")).toURL(); Document doc = Http.url(url).get(); Elements videos = doc.select("script"); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java index d4a0d8f3..d514c1e6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -82,12 +84,12 @@ public class TapasticRipper extends AbstractHTMLRipper { prefix.append(String.format("-%0" + imgLog + "dof%0" + imgLog + "d-", i + 1, images.size())); prefix.append(episode.filename.replace(" ", "-")); prefix.append("-"); - addURLToDownload(new URL(link), prefix.toString()); + addURLToDownload(new URI(link).toURL(), prefix.toString()); if (isThisATest()) { break; } } - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { LOGGER.error("[!] Exception while downloading " + url, e); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java index 31917199..49baa384 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TsuminoRipper.java @@ -1,9 +1,11 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -110,11 +112,11 @@ public class TsuminoRipper extends AbstractHTMLRipper { } @Override - public List getURLsFromPage(Document doc) { + public List getURLsFromPage(Document doc) throws UnsupportedEncodingException { JSONArray imageIds = getPageUrls(); List result = new ArrayList<>(); for (int i = 0; i < imageIds.length(); i++) { - result.add("http://www.tsumino.com/Image/Object?name=" + URLEncoder.encode(imageIds.getString(i))); + result.add("http://www.tsumino.com/Image/Object?name=" + URLEncoder.encode(imageIds.getString(i), StandardCharsets.UTF_8.name())); } return result; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java index e3888f75..baf5e212 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VidbleRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -70,11 +72,11 @@ public class VidbleRipper extends AbstractHTMLRipper { addURLToDownload(url, getPrefix(index)); } - public static List getURLsFromPage(URL url) throws IOException { + public static List getURLsFromPage(URL url) throws IOException, URISyntaxException { List urls = new ArrayList<>(); Document doc = Http.url(url).get(); for (String stringURL : getURLsFromPageStatic(doc)) { - urls.add(new URL(stringURL)); + urls.add(new URI(stringURL).toURL()); } return urls; } diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 85d092e8..8067cd92 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -84,7 +84,7 @@ public class RipUtils { try { logger.info("Getting vidble album " + url); result.addAll(VidbleRipper.getURLsFromPage(url)); - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { // Do nothing logger.warn("Exception while retrieving vidble page:", e); } @@ -94,7 +94,7 @@ public class RipUtils { try { logger.info("Getting eroshare album " + url); result.addAll(EroShareRipper.getURLs(url)); - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { // Do nothing logger.warn("Exception while retrieving eroshare page:", e); } From 95b0af4cfdf2acff3bb23063490d1c266f288536 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 28 Oct 2023 01:56:26 +0200 Subject: [PATCH 413/512] new URI instead of new URL. --- .../rarchives/ripme/ripper/rippers/SoundgasmRipper.java | 7 ++++--- .../ripme/tst/ripper/rippers/SoundgasmRipperTest.java | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java index 884b5bc9..106d0a6d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java @@ -1,13 +1,14 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -18,8 +19,8 @@ public class SoundgasmRipper extends AbstractHTMLRipper { private static final String HOST = "soundgasm.net"; - public SoundgasmRipper(URL url) throws IOException { - super(new URL(url.toExternalForm())); + public SoundgasmRipper(URL url) throws IOException, URISyntaxException { + super(new URI(url.toExternalForm()).toURL()); } @Override diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java index 877e9a04..847540a3 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java @@ -8,20 +8,21 @@ import org.junit.jupiter.api.Test; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; public class SoundgasmRipperTest extends RippersTest { @Test @Tag("flaky") public void testSoundgasmURLs() throws IOException, URISyntaxException { - SoundgasmRipper ripper = new SoundgasmRipper(new URI("https://soundgasm.net/u/_Firefly_xoxo/Rambles-with-my-Lovense").toURL()); + SoundgasmRipper ripper = new SoundgasmRipper(new URI("https://soundgasm.net/u/HTMLExamples/Making-Text-into-a-Soundgasm-Audio-Link").toURL()); testRipper(ripper); } @Test @Tag("flaky") public void testRedditSoundgasmURL() throws IOException, URISyntaxException { - RedditRipper ripper = new RedditRipper(new URI("https://www.reddit.com/r/gonewildaudio/comments/kn1bvj/f4m_mistress_controlled_my_lovense_while_i_tried/").toURL()); + RedditRipper ripper = new RedditRipper(new URI("https://www.reddit.com/user/Mistress_Minerva/").toURL()); testRipper(ripper); } } From 0c6f8d8516298516a2b6dc0df14b229857fba907 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 28 Oct 2023 01:56:41 +0200 Subject: [PATCH 414/512] new URI instead of new URL. --- .../com/rarchives/ripme/ripper/rippers/ArtstnRipper.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ArtstnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtstnRipper.java index 7505c078..1caeead4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ArtstnRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtstnRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import org.jsoup.Connection.Response; @@ -34,7 +36,7 @@ public class ArtstnRipper extends ArtStationRipper { if (artStationUrl == null) { throw new IOException("Null url received."); } - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { LOGGER.error("Couldnt resolve URL.", e); } @@ -42,7 +44,7 @@ public class ArtstnRipper extends ArtStationRipper { return super.getGID(artStationUrl); } - public URL getFinalUrl(URL url) throws IOException { + public URL getFinalUrl(URL url) throws IOException, URISyntaxException { if (url.getHost().endsWith("artstation.com")) { return url; } @@ -50,7 +52,7 @@ public class ArtstnRipper extends ArtStationRipper { LOGGER.info("Checking url: " + url); Response response = Http.url(url).connection().followRedirects(false).execute(); if (response.statusCode() / 100 == 3 && response.hasHeader("location")) { - return getFinalUrl(new URL(response.header("location"))); + return getFinalUrl(new URI(response.header("location")).toURL()); } else { return null; } From 194e5fdfb8c6eb86e96c1438017d9534050fa334 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 4 Nov 2023 09:29:04 +0100 Subject: [PATCH 415/512] delete duckmovies --- .../ripper/rippers/DuckmoviesRipper.java | 132 ------------------ .../ripper/rippers/DuckmoviesRipperTest.java | 21 --- 2 files changed, 153 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/DuckmoviesRipper.java delete mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/DuckmoviesRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DuckmoviesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DuckmoviesRipper.java deleted file mode 100644 index b9cb368b..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DuckmoviesRipper.java +++ /dev/null @@ -1,132 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - -import com.rarchives.ripme.ripper.AbstractSingleFileRipper; -import com.rarchives.ripme.utils.Http; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class DuckmoviesRipper extends AbstractSingleFileRipper { - public DuckmoviesRipper(URL url) throws IOException { - super(url); - } - - @Override - public boolean hasQueueSupport() { - return true; - } - - @Override - public boolean pageContainsAlbums(URL url) { - Pattern pa = Pattern.compile("https?://[a-zA-Z0-9]+.[a-zA-Z]+/(models|category)/([a-zA-Z0-9_-])+/?"); - Matcher ma = pa.matcher(url.toExternalForm()); - if (ma.matches()) { - return true; - } - pa = Pattern.compile("https?://[a-zA-Z0-9]+.[a-zA-Z]+/(models|category)/([a-zA-Z0-9_-])+/page/\\d+/?"); - ma = pa.matcher(url.toExternalForm()); - if (ma.matches()) { - return true; - } - return false; - } - - @Override - public List getAlbumsToQueue(Document doc) { - List urlsToAddToQueue = new ArrayList<>(); - for (Element elem : doc.select(".post > li > div > div > a")) { - urlsToAddToQueue.add(elem.attr("href")); - } - return urlsToAddToQueue; - } - - - private static List explicit_domains = Arrays.asList( - "vidporntube.fun", - "pornbj.fun", - "iwantporn.fun", - "neoporn.fun", - "yayporn.fun", - "freshporn.co", - "palapaja.stream", - "freshporn.co", - "pornvidx.fun", - "palapaja.com" - ); - - @Override - public String getHost() { - return url.toExternalForm().split("/")[2]; - } - - @Override - public String getDomain() { - return url.toExternalForm().split("/")[2]; - } - - @Override - public boolean canRip(URL url) { - String url_name = url.toExternalForm(); - return explicit_domains.contains(url_name.split("/")[2]); - } - - @Override - public List getURLsFromPage(Document doc) { - List results = new ArrayList<>(); - String duckMoviesUrl = doc.select("iframe").attr("src"); - try { - Document duckDoc = Http.url(new URL(duckMoviesUrl)).get(); - String videoURL = duckDoc.select("source").attr("src"); - // remove any white spaces so we can download the movie without a 400 error - videoURL = videoURL.replaceAll(" ", "%20"); - results.add(videoURL); - } catch (MalformedURLException e) { - LOGGER.error(duckMoviesUrl + " is not a valid url"); - } catch (IOException e) { - LOGGER.error("Unable to load page " + duckMoviesUrl); - e.printStackTrace(); - } - return results; - } - - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("https://[a-zA-Z0-9]+\\.[a-zA-Z]+/([a-zA-Z0-9\\-_]+)/?"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - p = Pattern.compile("https?://[a-zA-Z0-9]+.[a-zA-Z]+/(category|models)/([a-zA-Z0-9_-])+/?"); - m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - p = Pattern.compile("https?://[a-zA-Z0-9]+.[a-zA-Z]+/(category|models)/([a-zA-Z0-9_-])+/page/\\d+"); - m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - - throw new MalformedURLException( - "Expected duckmovies format:" - + "domain.tld/Video-title" - + " Got: " + url); - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, "", "", null, null, null); - } - - @Override - public boolean tryResumeDownload() {return true;} -} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DuckmoviesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DuckmoviesRipperTest.java deleted file mode 100644 index 909ddb09..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DuckmoviesRipperTest.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import com.rarchives.ripme.ripper.rippers.DuckmoviesRipper; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - -public class DuckmoviesRipperTest extends RippersTest { - @Test - @Disabled("Broken ripper") - public void testRip() throws IOException, URISyntaxException { - DuckmoviesRipper ripper = new DuckmoviesRipper( - new URI("https://palapaja.com/spyfam-stepbro-gives-in-to-stepsis-asian-persuasion/").toURL()); - testRipper(ripper); - } - -} \ No newline at end of file From 391b3696920d78479cd46dab94dfbba3f7fbf42c Mon Sep 17 00:00:00 2001 From: jpoulton Date: Sun, 29 Oct 2023 07:20:20 +0000 Subject: [PATCH 416/512] Support coomer.party --- .../ripper/rippers/CoomerPartyRipper.java | 152 ++++++++++++++++++ .../ripper/rippers/CoomerPartyRipperTest.java | 38 +++++ 2 files changed, 190 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/CoomerPartyRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java new file mode 100644 index 00000000..ca885da1 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java @@ -0,0 +1,152 @@ +package com.rarchives.ripme.ripper.rippers; + +import com.rarchives.ripme.ripper.AbstractJSONRipper; +import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.Utils; + +import org.apache.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * See this link for the API schema. + */ +public class CoomerPartyRipper extends AbstractJSONRipper { + private static final Logger LOGGER = Logger.getLogger(CoomerPartyRipper.class); + private static final String IMG_URL_BASE = "https://c3.coomer.su/data"; + private static final String VID_URL_BASE = "https://c1.coomer.su/data"; + private static final Pattern IMG_PATTERN = Pattern.compile("^.*\\.(jpg|jpeg|png|gif|apng|webp|tif|tiff)$", Pattern.CASE_INSENSITIVE); + private static final Pattern VID_PATTERN = Pattern.compile("^.*\\.(webm|mp4|m4v)$", Pattern.CASE_INSENSITIVE); + + // just so we can return a JSONObject from getFirstPage + private static final String KEY_WRAPPER_JSON_ARRAY = "array"; + + private static final String KEY_FILE = "file"; + private static final String KEY_PATH = "path"; + private static final String KEY_ATTACHMENTS = "attachments"; + + private final String service; + private final String user; + + public CoomerPartyRipper(URL url) throws IOException { + super(url); + List pathElements = Arrays.stream(url.getPath().split("/")) + .filter(element -> !element.isBlank()) + .collect(Collectors.toList()); + + service = pathElements.get(0); + user = pathElements.get(2); + + if (service == null || user == null || service.isBlank() || user.isBlank()) { + LOGGER.warn("service=" + service + ", user=" + user); + throw new MalformedURLException("Invalid coomer.party URL: " + url); + } + LOGGER.debug("Parsed service=" + service + " and user=" + user + " from " + url); + } + + @Override + protected String getDomain() { + return "coomer.party"; + } + + @Override + public String getHost() { + return "coomer.party"; + } + + @Override + public boolean canRip(URL url) { + String host = url.getHost(); + return host.endsWith("coomer.party") || host.endsWith("coomer.su"); + } + + @Override + public String getGID(URL url) { + return Utils.filesystemSafe(String.format("%s_%s", service, user)); + } + + @Override + protected JSONObject getFirstPage() throws IOException { + String apiUrl = String.format("https://coomer.su/api/v1/%s/user/%s", service, user); + String jsonArrayString = Http.url(apiUrl) + .ignoreContentType() + .response() + .body(); + JSONArray jsonArray = new JSONArray(jsonArrayString); + + // Ideally we'd just return the JSONArray from here, but we have to wrap it in a JSONObject + JSONObject wrapperObject = new JSONObject(); + wrapperObject.put(KEY_WRAPPER_JSON_ARRAY, jsonArray); + return wrapperObject; + } + + @Override + protected List getURLsFromJSON(JSONObject json) { + // extract the array from our wrapper JSONObject + JSONArray posts = json.getJSONArray(KEY_WRAPPER_JSON_ARRAY); + ArrayList urls = new ArrayList<>(); + for (int i = 0; i < posts.length(); i++) { + JSONObject post = posts.getJSONObject(i); + pullFileUrl(post, urls); + pullAttachmentUrls(post, urls); + } + LOGGER.debug("Pulled " + urls.size() + " URLs from " + posts.length() + " posts"); + return urls; + } + + @Override + protected void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + + private void pullFileUrl(JSONObject post, ArrayList results) { + try { + JSONObject file = post.getJSONObject(KEY_FILE); + String path = file.getString(KEY_PATH); + if (isImage(path)) { + String url = IMG_URL_BASE + path; + results.add(url); + } else if (isVideo(path)) { + String url = VID_URL_BASE + path; + results.add(url); + } else { + LOGGER.error("Unknown extension for coomer.su path: " + path); + } + } catch (JSONException e) { + /* No-op */ + } + } + + private void pullAttachmentUrls(JSONObject post, ArrayList results) { + try { + JSONArray attachments = post.getJSONArray(KEY_ATTACHMENTS); + for (int i = 0; i < attachments.length(); i++) { + JSONObject attachment = attachments.getJSONObject(0); + pullFileUrl(attachment, results); + } + } catch (JSONException e) { + /* No-op */ + } + } + + private boolean isImage(String path) { + Matcher matcher = IMG_PATTERN.matcher(path); + return matcher.matches(); + } + + private boolean isVideo(String path) { + Matcher matcher = VID_PATTERN.matcher(path); + return matcher.matches(); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CoomerPartyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CoomerPartyRipperTest.java new file mode 100644 index 00000000..ccc26a76 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CoomerPartyRipperTest.java @@ -0,0 +1,38 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.rarchives.ripme.ripper.rippers.CoomerPartyRipper; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URL; + +public class CoomerPartyRipperTest extends RippersTest { + @Test + public void testRip() throws IOException { + URL url = new URL("https://coomer.su/onlyfans/user/soogsx"); + CoomerPartyRipper ripper = new CoomerPartyRipper(url); + testRipper(ripper); + } + + @Test + public void testUrlParsing() throws IOException { + String expectedGid = "onlyfans_soogsx"; + String[] urls = new String[]{ + "https://coomer.su/onlyfans/user/soogsx", // normal url + "http://coomer.su/onlyfans/user/soogsx", // http, not https + "https://coomer.su/onlyfans/user/soogsx/", // with slash at the end + "https://coomer.su/onlyfans/user/soogsx?whatever=abc", // with url params + "https://coomer.party/onlyfans/user/soogsx", // alternate domain + }; + for (String stringUrl : urls) { + URL url = new URL(stringUrl); + CoomerPartyRipper ripper = new CoomerPartyRipper(url); + assertTrue(ripper.canRip(url)); + assertEquals(expectedGid, ripper.getGID(url)); + } + } +} \ No newline at end of file From 1e864b7814a4cc4995d02f9e6592b9ebf61f7699 Mon Sep 17 00:00:00 2001 From: jpoulton Date: Sun, 29 Oct 2023 07:21:12 +0000 Subject: [PATCH 417/512] Update .gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 896c43fe..fe1e80c6 100644 --- a/.gitignore +++ b/.gitignore @@ -111,6 +111,11 @@ $RECYCLE.BIN/ .vscode .idea .project +local.properties + +### Build files +.gradle/ +build/ ### Ripme ### ripme.log From 2e3dba701ef8c2f4d78842806a4a3b5fdefd90e4 Mon Sep 17 00:00:00 2001 From: jpoulton Date: Sun, 29 Oct 2023 07:22:22 +0000 Subject: [PATCH 418/512] Update CoomerPartyRipper.java --- .../com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java index ca885da1..5248d633 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java @@ -36,7 +36,10 @@ public class CoomerPartyRipper extends AbstractJSONRipper { private static final String KEY_PATH = "path"; private static final String KEY_ATTACHMENTS = "attachments"; + // One of "onlyfans" or "fansly", but might have others in future? private final String service; + + // Username of the page to be ripped private final String user; public CoomerPartyRipper(URL url) throws IOException { From 6d7503facb3bd81e63dd0c0eac4252081f449e2d Mon Sep 17 00:00:00 2001 From: jpoulton Date: Mon, 30 Oct 2023 05:51:45 +0000 Subject: [PATCH 419/512] Use correct index --- .../com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java index 5248d633..e15bb643 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java @@ -135,7 +135,7 @@ public class CoomerPartyRipper extends AbstractJSONRipper { try { JSONArray attachments = post.getJSONArray(KEY_ATTACHMENTS); for (int i = 0; i < attachments.length(); i++) { - JSONObject attachment = attachments.getJSONObject(0); + JSONObject attachment = attachments.getJSONObject(i); pullFileUrl(attachment, results); } } catch (JSONException e) { From ce0e60c501841432d0e1404c816c52e3ccb7f0db Mon Sep 17 00:00:00 2001 From: Georgi Marinov Date: Sat, 4 Nov 2023 17:29:48 +0200 Subject: [PATCH 420/512] LusciousRipper fixed --- .../ripme/ripper/rippers/LusciousRipper.java | 142 ++++++------------ 1 file changed, 45 insertions(+), 97 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java index 099eaf73..de97c533 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java @@ -1,27 +1,26 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; +import org.json.JSONArray; +import org.json.JSONObject; +import org.jsoup.Connection; import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; import java.net.URL; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; public class LusciousRipper extends AbstractHTMLRipper { - private static final int RETRY_COUNT = 5; // Keeping it high for read timeout exception. + private static String albumid; private static final Pattern P = Pattern.compile("^https?://(?:members\\.|legacy\\.|www\\.)?luscious.net/albums/([-_.0-9a-zA-Z]+)/?"); - private final DownloadThreadPool lusciousThreadPool = new DownloadThreadPool("lusciousThreadPool"); public LusciousRipper(URL url) throws IOException { super(url); @@ -38,37 +37,48 @@ public class LusciousRipper extends AbstractHTMLRipper { } @Override - public Document getFirstPage() throws IOException { - return super.getFirstPage(); - } - - @Override - public List getURLsFromPage(Document page) { + public List getURLsFromPage(Document page) { // gets urls for all pages through the api List urls = new ArrayList<>(); - Elements urlElements = page.select("div.item.thumbnail.ic_container > a"); - for (Element e : urlElements) { - urls.add(e.attr("abs:href")); + int totalPages = 1; + + for (int i = 1; i <= totalPages; i++) { + String APIStringWOVariables = "https://apicdn.luscious.net/graphql/nobatch/?operationName=PictureListInsideAlbum&query=%2520query%2520PictureListInsideAlbum%28%2524input%253A%2520PictureListInput%21%29%2520%257B%2520picture%2520%257B%2520list%28input%253A%2520%2524input%29%2520%257B%2520info%2520%257B%2520...FacetCollectionInfo%2520%257D%2520items%2520%257B%2520__typename%2520id%2520title%2520description%2520created%2520like_status%2520number_of_comments%2520number_of_favorites%2520moderation_status%2520width%2520height%2520resolution%2520aspect_ratio%2520url_to_original%2520url_to_video%2520is_animated%2520position%2520permissions%2520url%2520tags%2520%257B%2520category%2520text%2520url%2520%257D%2520thumbnails%2520%257B%2520width%2520height%2520size%2520url%2520%257D%2520%257D%2520%257D%2520%257D%2520%257D%2520fragment%2520FacetCollectionInfo%2520on%2520FacetCollectionInfo%2520%257B%2520page%2520has_next_page%2520has_previous_page%2520total_items%2520total_pages%2520items_per_page%2520url_complete%2520%257D%2520&variables="; + Connection con = Http.url(APIStringWOVariables + encodeVariablesPartOfURL(i, albumid)).method(Connection.Method.GET).retries(5).connection(); + con.ignoreHttpErrors(true); + con.ignoreContentType(true); + con.userAgent("Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/119.0"); + Connection.Response res; + try { + res = con.execute(); + } catch (IOException e) { + throw new RuntimeException(e); + } + String body = res.body(); + + JSONObject jsonObject = new JSONObject(body); + + JSONObject data = jsonObject.getJSONObject("data"); + JSONObject picture = data.getJSONObject("picture"); + JSONObject list = picture.getJSONObject("list"); + JSONArray items = list.getJSONArray("items"); + JSONObject info = list.getJSONObject("info"); + totalPages = info.getInt("total_pages"); + + for (int j = 0; j < items.length(); j++) { + JSONObject item = items.getJSONObject(j); + String urlToOriginal = item.getString("url_to_original"); + urls.add(urlToOriginal); + } } return urls; } - @Override - public Document getNextPage(Document doc) throws IOException { - // luscious sends xhr requests to nextPageUrl and appends new set of images to the current page while in browser. - // Simply GET the nextPageUrl also works. Therefore, we do this... - Element nextPageElement = doc.select("div#next_page > div > a").first(); - if (nextPageElement == null) { - throw new IOException("No next page found."); - } - - return Http.url(nextPageElement.attr("abs:href")).get(); - } - @Override public String getGID(URL url) throws MalformedURLException { Matcher m = P.matcher(url.toExternalForm()); if (m.matches()) { + albumid = m.group(1).split("_")[m.group(1).split("_").length - 1]; return m.group(1); } throw new MalformedURLException("Expected luscious.net URL format: " @@ -76,79 +86,17 @@ public class LusciousRipper extends AbstractHTMLRipper { } @Override - public void downloadURL(URL url, int index) { - lusciousThreadPool.addThread(new LusciousDownloadThread(url, index)); + protected void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), null); } - @Override - public DownloadThreadPool getThreadPool() { - return lusciousThreadPool; - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { - // Sanitizes the url removing GET parameters and convert to legacy api url. - // "https://legacy.luscious.net/albums/albumname" + public static String encodeVariablesPartOfURL(int page, String albumId) { try { - Matcher m = P.matcher(url.toString()); - if (m.matches()) { - String sanitizedUrl = m.group(); - sanitizedUrl = sanitizedUrl.replaceFirst( - "^https?://(?:members\\.|legacy\\.|www\\.)?luscious.net", - "https://legacy.luscious.net"); - return new URI(sanitizedUrl).toURL(); - } + String json = "{\"input\":{\"filters\":[{\"name\":\"album_id\",\"value\":\"" + albumId + "\"}],\"display\":\"rating_all_time\",\"items_per_page\":50,\"page\":" + page + "}}"; - throw new Exception("ERROR: Unable to sanitize url."); - } catch (Exception e) { - LOGGER.info("Error sanitizing the url."); - LOGGER.error(e); - return super.sanitizeURL(url); + return URLEncoder.encode(json, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("Could not encode variables"); } } - - @Override - public String normalizeUrl(String url) { - try { - return url.replaceFirst( - "^https?://(?:members\\.|legacy\\.)?luscious.net", "https://www.luscious.net"); - } catch (Exception e) { - LOGGER.info("Error normalizing the url."); - LOGGER.error(e); - return super.normalizeUrl(url); - } - } - - public class LusciousDownloadThread implements Runnable { - private final URL url; - private final int index; - - public LusciousDownloadThread(URL url, int index) { - this.url = url; - this.index = index; - } - - @Override - public void run() { - try { - Document page = Http.url(url).retries(RETRY_COUNT).get(); - - String downloadUrl = page.select(".icon-download").attr("abs:href"); - if (downloadUrl.equals("")) { - // This is here for pages with mp4s instead of images. - downloadUrl = page.select("div > video > source").attr("src"); - if (!downloadUrl.equals("")) { - throw new IOException("Could not find download url for image or video."); - } - } - - //If a valid download url was found. - addURLToDownload(new URI(downloadUrl).toURL(), getPrefix(index)); - - } catch (IOException | URISyntaxException e) { - LOGGER.error("Error downloadiong url " + url, e); - } - } - - } } From ea8e4cdfeafc9ca108269301ede1f201112206ea Mon Sep 17 00:00:00 2001 From: jpoulton Date: Sat, 4 Nov 2023 15:45:39 +0000 Subject: [PATCH 421/512] Skip specified URL file extensions --- .../rarchives/ripme/ripper/AbstractHTMLRipper.java | 3 +++ .../rarchives/ripme/ripper/AbstractJSONRipper.java | 3 +++ .../com/rarchives/ripme/ripper/AbstractRipper.java | 14 ++++++++++++++ .../com/rarchives/ripme/ripper/AlbumRipper.java | 3 +++ .../com/rarchives/ripme/ripper/VideoRipper.java | 3 +++ src/main/resources/rip.properties | 3 +++ 6 files changed, 29 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 6380a1c6..7f3509b1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -342,6 +342,9 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); return false; } + if (shouldIgnoreURL(url)) { + return false; + } if (Utils.getConfigBoolean("urls_only.save", false)) { // Output URL to file Path urlFile = Paths.get(this.workingDir + "/urls.txt"); diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 523a8ab8..31f94cde 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -160,6 +160,9 @@ public abstract class AbstractJSONRipper extends AbstractRipper { LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); return false; } + if (shouldIgnoreURL(url)) { + return false; + } if (Utils.getConfigBoolean("urls_only.save", false)) { // Output URL to file Path urlFile = Paths.get(this.workingDir + "/urls.txt"); diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 1a25af18..fa58b5c1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -699,4 +699,18 @@ public abstract class AbstractRipper protected boolean useByteProgessBar() { return false;} // If true ripme will try to resume a broken download for this ripper protected boolean tryResumeDownload() { return false;} + + protected boolean shouldIgnoreURL(URL url) { + final String[] ignoredExtensions = Utils.getConfigStringArray("download.ignore_extensions"); + if (ignoredExtensions == null || ignoredExtensions.length == 0) return false; // nothing ignored + String[] pathElements = url.getPath().split("\\."); + if (pathElements.length == 0) return false; // no extension, can't filter + String extension = pathElements[pathElements.length - 1]; + for (String ignoredExtension : ignoredExtensions) { + if (ignoredExtension.equalsIgnoreCase(extension)) { + return true; + } + } + return false; + } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 96c352f5..94d506e3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -70,6 +70,9 @@ public abstract class AlbumRipper extends AbstractRipper { LOGGER.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); return false; } + if (shouldIgnoreURL(url)) { + return false; + } if (Utils.getConfigBoolean("urls_only.save", false)) { // Output URL to file Path urlFile = Paths.get(this.workingDir + "/urls.txt"); diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index cd4c95b3..391ce2f4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -68,6 +68,9 @@ public abstract class VideoRipper extends AbstractRipper { this.url = url; return true; } + if (shouldIgnoreURL(url)) { + return false; + } threadPool.addThread(new DownloadVideoThread(url, saveAs, this)); } return true; diff --git a/src/main/resources/rip.properties b/src/main/resources/rip.properties index cac0c1f1..35d6c123 100644 --- a/src/main/resources/rip.properties +++ b/src/main/resources/rip.properties @@ -17,6 +17,9 @@ page.timeout = 5000 # Maximum size of downloaded files in bytes (required) download.max_size = 104857600 +# Any URLs ending with one of these comma-separated values will be skipped +#download.ignore_extensions = mp4,gif,m4v,webm,html + # Don't retry on 404 errors error.skip404 = true From 154cf536cebaed50f4a7a28ae679607a1ac13f17 Mon Sep 17 00:00:00 2001 From: jpoulton Date: Sat, 4 Nov 2023 15:45:53 +0000 Subject: [PATCH 422/512] Pass skip status to the UI log --- .../java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 1 + .../java/com/rarchives/ripme/ripper/AbstractJSONRipper.java | 1 + src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java | 1 + src/main/java/com/rarchives/ripme/ripper/VideoRipper.java | 1 + src/main/java/com/rarchives/ripme/ui/MainWindow.java | 5 +++++ src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java | 1 + 6 files changed, 10 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 7f3509b1..c906f9da 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -343,6 +343,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { return false; } if (shouldIgnoreURL(url)) { + sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); return false; } if (Utils.getConfigBoolean("urls_only.save", false)) { diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 31f94cde..71b19986 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -161,6 +161,7 @@ public abstract class AbstractJSONRipper extends AbstractRipper { return false; } if (shouldIgnoreURL(url)) { + sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); return false; } if (Utils.getConfigBoolean("urls_only.save", false)) { diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 94d506e3..12139cf3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -71,6 +71,7 @@ public abstract class AlbumRipper extends AbstractRipper { return false; } if (shouldIgnoreURL(url)) { + sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); return false; } if (Utils.getConfigBoolean("urls_only.save", false)) { diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 391ce2f4..014998fa 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -69,6 +69,7 @@ public abstract class VideoRipper extends AbstractRipper { return true; } if (shouldIgnoreURL(url)) { + sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); return false; } threadPool.addThread(new DownloadVideoThread(url, saveAs, this)); diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 357ee6e9..a6234333 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1435,6 +1435,11 @@ public final class MainWindow implements Runnable, RipStatusHandler { appendLog((String) msg.getObject(), Color.ORANGE); } break; + case DOWNLOAD_SKIP: + if (LOGGER.isEnabled(Level.INFO)) { + appendLog((String) msg.getObject(), Color.YELLOW); + } + break; case RIP_ERRORED: if (LOGGER.isEnabled(Level.ERROR)) { diff --git a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java index 207968d9..f589e9db 100644 --- a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java +++ b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java @@ -13,6 +13,7 @@ public class RipStatusMessage { DOWNLOAD_COMPLETE_HISTORY("Download Complete History"), RIP_COMPLETE("Rip Complete"), DOWNLOAD_WARN("Download problem"), + DOWNLOAD_SKIP("Download Skipped"), TOTAL_BYTES("Total bytes"), COMPLETED_BYTES("Completed bytes"), RIP_ERRORED("Rip Errored"), From d82cae1b78c29e3885f1643a62c46ea0121fb9c2 Mon Sep 17 00:00:00 2001 From: jpoulton Date: Sat, 4 Nov 2023 15:46:12 +0000 Subject: [PATCH 423/512] Fix bug in CoomerPartyRipper --- .../rarchives/ripme/ripper/rippers/CoomerPartyRipper.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java index e15bb643..e2dd301e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java @@ -4,7 +4,8 @@ import com.rarchives.ripme.ripper.AbstractJSONRipper; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -23,7 +24,7 @@ import java.util.stream.Collectors; * See this link for the API schema. */ public class CoomerPartyRipper extends AbstractJSONRipper { - private static final Logger LOGGER = Logger.getLogger(CoomerPartyRipper.class); + private static final Logger LOGGER = LogManager.getLogger(CoomerPartyRipper.class); private static final String IMG_URL_BASE = "https://c3.coomer.su/data"; private static final String VID_URL_BASE = "https://c1.coomer.su/data"; private static final Pattern IMG_PATTERN = Pattern.compile("^.*\\.(jpg|jpeg|png|gif|apng|webp|tif|tiff)$", Pattern.CASE_INSENSITIVE); From 4a61ff97e5d15b9722bbdec8cd8bd665515b8037 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 18 Nov 2023 05:12:33 +0100 Subject: [PATCH 424/512] coomerPartyrippertest new URI instead of new URL --- .../tst/ripper/rippers/CoomerPartyRipperTest.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CoomerPartyRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CoomerPartyRipperTest.java index ccc26a76..c35822a8 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CoomerPartyRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CoomerPartyRipperTest.java @@ -8,18 +8,20 @@ import com.rarchives.ripme.ripper.rippers.CoomerPartyRipper; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; public class CoomerPartyRipperTest extends RippersTest { @Test - public void testRip() throws IOException { - URL url = new URL("https://coomer.su/onlyfans/user/soogsx"); + public void testRip() throws IOException, URISyntaxException { + URL url = new URI("https://coomer.su/onlyfans/user/soogsx").toURL(); CoomerPartyRipper ripper = new CoomerPartyRipper(url); testRipper(ripper); } @Test - public void testUrlParsing() throws IOException { + public void testUrlParsing() throws IOException, URISyntaxException { String expectedGid = "onlyfans_soogsx"; String[] urls = new String[]{ "https://coomer.su/onlyfans/user/soogsx", // normal url @@ -29,7 +31,7 @@ public class CoomerPartyRipperTest extends RippersTest { "https://coomer.party/onlyfans/user/soogsx", // alternate domain }; for (String stringUrl : urls) { - URL url = new URL(stringUrl); + URL url = new URI(stringUrl).toURL(); CoomerPartyRipper ripper = new CoomerPartyRipper(url); assertTrue(ripper.canRip(url)); assertEquals(expectedGid, ripper.getGID(url)); From 37ddf5575fba4c9f0049d1af155a4c2ae58cac00 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 18 Nov 2023 05:19:30 +0100 Subject: [PATCH 425/512] testHentaifoundry flaky, prevents rollout --- .../ripme/tst/ripper/rippers/HentaifoundryRipperTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java index 11101bdc..a0ea694e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java @@ -6,6 +6,7 @@ import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.HentaifoundryRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class HentaifoundryRipperTest extends RippersTest { @@ -15,12 +16,14 @@ public class HentaifoundryRipperTest extends RippersTest { testRipper(ripper); } @Test + @Tag("flaky") public void testHentaifoundryGetGID() throws IOException, URISyntaxException { HentaifoundryRipper ripper = new HentaifoundryRipper(new URI("https://www.hentai-foundry.com/stories/user/Rakked").toURL()); testRipper(ripper); Assertions.assertEquals("Rakked", ripper.getGID(new URI("https://www.hentai-foundry.com/stories/user/Rakked").toURL())); } @Test + @Tag("flaky") public void testHentaifoundryPdfRip() throws IOException, URISyntaxException { HentaifoundryRipper ripper = new HentaifoundryRipper(new URI("https://www.hentai-foundry.com/stories/user/Rakked").toURL()); testRipper(ripper); From 8897842b55583c87102cf8c426f912c2ef4e21d4 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 18 Nov 2023 06:54:16 +0100 Subject: [PATCH 426/512] exec stringarray instead of string, java-21 deprecation. --- .../java/com/rarchives/ripme/ui/MainWindow.java | 15 ++++++++------- .../com/rarchives/ripme/ui/UpdateUtils.java | 17 +++++++++-------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index a6234333..6cd5c574 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -470,7 +470,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { queuePanel.setBorder(emptyBorder); queuePanel.setVisible(false); queuePanel.setPreferredSize(new Dimension(300, 250)); - queueListModel = new DefaultListModel(); + queueListModel = new DefaultListModel<>(); JList queueList = new JList(queueListModel); queueList.setSelectionMode(ListSelectionModel.MULTIPLE_INTERVAL_SELECTION); QueueMenuMouseListener queueMenuMouseListener; @@ -732,7 +732,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { private void update() { try { String urlText = ripTextfield.getText().trim(); - if (urlText.equals("")) { + if (urlText.isEmpty()) { return; } if (!urlText.startsWith("http")) { @@ -1499,12 +1499,13 @@ public final class MainWindow implements Runnable, RipStatusHandler { */ if (Utils.getConfigBoolean("enable.finish.command", false)) { try { - String commandToRun = Utils.getConfigString("finish.command", "ls"); - commandToRun = commandToRun.replaceAll("%url%", url); - commandToRun = commandToRun.replaceAll("%path%", f.toAbsolutePath().toString()); + String cmdStr = Utils.getConfigString("finish.command", "ls"); + cmdStr = cmdStr.replaceAll("%url%", url); + cmdStr = cmdStr.replaceAll("%path%", f.toAbsolutePath().toString()); + // java dropped the exec string executor, as the string is only split very trivial. + // do the same at the moment, and split, to get rid of java-21 deprecation warning. + String[] commandToRun = cmdStr.split(" "); LOGGER.info("RUnning command " + commandToRun); - // code from: - // https://stackoverflow.com/questions/5711084/java-runtime-getruntime-getting-output-from-executing-a-command-line-program Process proc = Runtime.getRuntime().exec(commandToRun); BufferedReader stdInput = new BufferedReader(new InputStreamReader(proc.getInputStream())); diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 4d2c7cac..128eabba 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -222,13 +222,14 @@ public class UpdateUtils { public static String createSha256(Path file) { try { MessageDigest digest = MessageDigest.getInstance("SHA-256"); - InputStream fis = Files.newInputStream(file); - int n = 0; - byte[] buffer = new byte[8192]; - while (n != -1) { - n = fis.read(buffer); - if (n > 0) { - digest.update(buffer, 0, n); + try (InputStream fis = Files.newInputStream(file)) { + int n = 0; + byte[] buffer = new byte[8192]; + while (n != -1) { + n = fis.read(buffer); + if (n > 0) { + digest.update(buffer, 0, n); + } } } byte[] hash = digest.digest(); @@ -313,7 +314,7 @@ public class UpdateUtils { if (shouldLaunch) { // No need to do it during shutdown: the file used will indeed be the new one logger.info("Executing: " + mainFile); - Runtime.getRuntime().exec("java -jar " + mainFile); + Runtime.getRuntime().exec(new String[]{"java", "-jar", mainFile.toString()}); } logger.info("Update installed, newer version should be executed upon relaunch"); System.exit(0); From 14c0779b7d4059b376ec60a2e3d5ede26444938a Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 18 Nov 2023 06:55:17 +0100 Subject: [PATCH 427/512] new URI instead of new URL, photobucketrupper. --- .../rarchives/ripme/ripper/rippers/PhotobucketRipper.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PhotobucketRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PhotobucketRipper.java index 680d2c09..097fe2c0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PhotobucketRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PhotobucketRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -82,7 +84,7 @@ public class PhotobucketRipper extends AbstractHTMLRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { LOGGER.info(url); String u = url.toExternalForm(); if (u.contains("?")) { @@ -93,11 +95,11 @@ public class PhotobucketRipper extends AbstractHTMLRipper { // append trailing slash u = u + "/"; } - return new URL(u); + return new URI(u).toURL(); } @Override - public String getGID(URL url) throws MalformedURLException { + public String getGID(URL url) throws MalformedURLException, URISyntaxException { Matcher m; URL sanitized = sanitizeURL(url); From f54aa6f833d189d22c93fb43bceab6b63ebb05aa Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 18 Nov 2023 06:56:34 +0100 Subject: [PATCH 428/512] new URI instead of new URL, abstractjsonripper. --- .../java/com/rarchives/ripme/ripper/AbstractJSONRipper.java | 3 ++- .../com/rarchives/ripme/ripper/rippers/DerpiRipper.java | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 71b19986..2fbc05e1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -8,6 +8,7 @@ import org.json.JSONObject; import java.io.File; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -56,7 +57,7 @@ public abstract class AbstractJSONRipper extends AbstractRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { return url; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DerpiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DerpiRipper.java index 1feaf692..c2acdc56 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DerpiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DerpiRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -37,7 +39,7 @@ public class DerpiRipper extends AbstractJSONRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { String u = url.toExternalForm(); String[] uu = u.split("\\?", 2); String newU = uu[0]; @@ -54,7 +56,7 @@ public class DerpiRipper extends AbstractJSONRipper { newU += "&key=" + key; } - return new URL(newU); + return new URI(newU).toURL(); } @Override From 2ece00882f9f140eab3f1e9657373692bc0cad7c Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 18 Nov 2023 07:01:48 +0100 Subject: [PATCH 429/512] new URI instead of new URL, abstractjsonripper nextPage. --- .../java/com/rarchives/ripme/ripper/AbstractJSONRipper.java | 4 ++-- .../java/com/rarchives/ripme/ripper/rippers/DerpiRipper.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 2fbc05e1..538a4b5d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -38,7 +38,7 @@ public abstract class AbstractJSONRipper extends AbstractRipper { public abstract String getHost(); protected abstract JSONObject getFirstPage() throws IOException; - protected JSONObject getNextPage(JSONObject doc) throws IOException { + protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException { throw new IOException("getNextPage not implemented"); } protected abstract List getURLsFromJSON(JSONObject json); @@ -104,7 +104,7 @@ public abstract class AbstractJSONRipper extends AbstractRipper { try { sendUpdate(STATUS.LOADING_RESOURCE, "next page"); json = getNextPage(json); - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { LOGGER.info("Can't get next page: " + e.getMessage()); break; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DerpiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DerpiRipper.java index c2acdc56..a0538614 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DerpiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DerpiRipper.java @@ -101,10 +101,10 @@ public class DerpiRipper extends AbstractJSONRipper { } @Override - public JSONObject getNextPage(JSONObject doc) throws IOException { + public JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException { currPage++; String u = currUrl.toExternalForm() + "&page=" + Integer.toString(currPage); - JSONObject json = Http.url(new URL(u)).getJSON(); + JSONObject json = Http.url(new URI(u).toURL()).getJSON(); JSONArray arr; if (json.has("images")) { arr = json.getJSONArray("images"); From 073eb5af186d63336d241833f3c32fc22eed6948 Mon Sep 17 00:00:00 2001 From: joroto Date: Sun, 26 Nov 2023 06:52:22 -0800 Subject: [PATCH 430/512] LusciousRipper get next page test removed --- .../tst/ripper/rippers/LusciousRipperTest.java | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java index 25dcb94d..4cfffcb8 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java @@ -26,20 +26,4 @@ public class LusciousRipperTest extends RippersTest { LusciousRipper ripper = new LusciousRipper(url); Assertions.assertEquals("h-na-alice-wa-suki-desu-ka-do-you-like-alice-when_321609", ripper.getGID(url)); } - - @Test - @Disabled("test or ripper broken") - public void testGetNextPage() throws IOException, URISyntaxException { - URL multiPageAlbumUrl = new URI("https://luscious.net/albums/women-of-color_58/").toURL(); - LusciousRipper multiPageRipper = new LusciousRipper(multiPageAlbumUrl); - assert (multiPageRipper.getNextPage(multiPageRipper.getFirstPage()) != null); - - URL singlePageAlbumUrl = new URI("https://members.luscious.net/albums/bakaneko-navidarks_332097/").toURL(); - LusciousRipper singlePageRipper = new LusciousRipper(singlePageAlbumUrl); - try { - singlePageRipper.getNextPage(singlePageRipper.getFirstPage()); - } catch (IOException e) { - Assertions.assertEquals("No next page found.", e.getMessage()); - } - } } \ No newline at end of file From 63760ea9b48f44f6e1e6cd2d8b57ee039f5a307c Mon Sep 17 00:00:00 2001 From: joroto Date: Sun, 26 Nov 2023 07:02:59 -0800 Subject: [PATCH 431/512] Refactoring --- .../rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java index 4cfffcb8..bc8594d6 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/LusciousRipperTest.java @@ -13,7 +13,7 @@ import org.junit.jupiter.api.Test; public class LusciousRipperTest extends RippersTest { @Test @Disabled("test or ripper broken") - public void testPahealRipper() throws IOException, URISyntaxException { + public void testLusciousRipper() throws IOException, URISyntaxException { // a photo set LusciousRipper ripper = new LusciousRipper( new URI("https://luscious.net/albums/h-na-alice-wa-suki-desu-ka-do-you-like-alice-when_321609/").toURL()); From b080faaed86680f4f82eb91e19221f991963f1a9 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 1 Dec 2023 16:57:10 +0100 Subject: [PATCH 432/512] flaky test hentai, femjoy --- .../ripme/tst/ripper/rippers/FemjoyhunterRipperTest.java | 2 ++ .../ripme/tst/ripper/rippers/HentaifoundryRipperTest.java | 1 + 2 files changed, 3 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemjoyhunterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemjoyhunterRipperTest.java index ce1a221f..3f295a2c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemjoyhunterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FemjoyhunterRipperTest.java @@ -6,10 +6,12 @@ import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.FemjoyhunterRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class FemjoyhunterRipperTest extends RippersTest { @Test + @Tag("flaky") public void testRip() throws IOException, URISyntaxException { FemjoyhunterRipper ripper = new FemjoyhunterRipper(new URI( "https://www.femjoyhunter.com/alisa-i-got-nice-big-breasts-and-fine-ass-so-she-seems-to-be-a-hottest-brunette-5936/").toURL()); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java index a0ea694e..2e360a9c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaifoundryRipperTest.java @@ -11,6 +11,7 @@ import org.junit.jupiter.api.Test; public class HentaifoundryRipperTest extends RippersTest { @Test + @Tag("flaky") public void testHentaifoundryRip() throws IOException, URISyntaxException { HentaifoundryRipper ripper = new HentaifoundryRipper(new URI("https://www.hentai-foundry.com/pictures/user/personalami").toURL()); testRipper(ripper); From 8eb9954cb6b2b4021ce9bbe27ecdf5ae24e4060f Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 1 Dec 2023 17:16:21 +0100 Subject: [PATCH 433/512] release 2.1.7 --- ripme.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ripme.json b/ripme.json index 9d23c24a..e856cbaa 100644 --- a/ripme.json +++ b/ripme.json @@ -1,7 +1,8 @@ { - "latestVersion": "2.1.6-1-68189f27", - "currentHash": "adffec078cc6e1da42699f874176744622a00a1dace000f2ef0c7dad28957faa", + "latestVersion": "2.1.7-29-b080faae", + "currentHash": "a2bedd4d524efd98884dad38c888059ea5bbc113924eefa73f5893189beab075", "changeList": [ + "2.1.7-29-b080faae: luciousripper fix, java-21 adjustments.", "2.1.6-1-68189f27: erome fix.", "2.1.5-8-ba51d7b: ripme running with java-17.", "2.1.4-38-836a7494: fixed imagefap ripper.", From 658241970dbc000e00723fc92fbbdea272487af0 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 1 Dec 2023 17:22:32 +0100 Subject: [PATCH 434/512] gradle-8.5 update --- gradle/wrapper/gradle-wrapper.jar | Bin 58910 -> 63721 bytes gradle/wrapper/gradle-wrapper.properties | 4 +- gradlew | 294 ++++++++++++++--------- gradlew.bat | 34 +-- 4 files changed, 193 insertions(+), 139 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 62d4c053550b91381bbd28b1afc82d634bf73a8a..7f93135c49b765f8051ef9d0a6055ff8e46073d8 100644 GIT binary patch delta 44733 zcmZ6yQ

+(5+jxZQC}wY}>Z&db@1fwr$(C)n%K#d!K)tefD>5R<6vkF4jdv#*;B; z=1zm2--CiH%7B8QCFG+g;GhDdB``Ssd*qV>qb1ERVJGY$|L+-k5*RQ(CK?bB6ci8; zkPuMl*_sC>3=oh$CJ+!!Qi2dh()b!CAV(9@7k35!N1yKV_;EyJRvAM&d$^Gm4mw~M z8anoR9fBMoHEB6GQTHl}p5#1xiCW&$N(M8>bphCQQHZlZgrgmJa}n6is=x=a^EEg5 z#mA$u+G*V$(06>k`}h0RcMeg|>(&p5BkCDOzx>d7yI6Q6>iEI9qg!-9Mom=`;0Uf? zx_<&=|8j)x->nRyUvjwO8AI6icCQUyr=moOe0k^X^#+07cE|zVR;76yxL1BC=hl=#Kn!?2ClFb3)JSr$px~-&dj0 zzPCc>JY@CvRY&1F9vgxYpcE8jMC{8yMD}ffF&Hd@AxKI$NTq}Rf$615tE+h9=`A?S zdP@y-s0!cb1&QkV*yJI}kw6sZ{fu>gy}9Ju^vUll^XU6U{Wdx(B0RRE1s zdz8-4b-7-E3TzED(Q&-od0sY` zf>f%$C7sBd!wfe9lv~LQZnJEBT!dNT8tdqkcrk%BSwiAzQx#bU&^g}1Xg799zvO6D zHg!pqa?OF-+(crJfGA^@<|Rgah?iJP0ejA80#|C6^k&n2hlpuBAzCabKODN0${>eg znHf;cPngpDL+nC_&G}3mkfLwOkH6u9y*tYq6ne=4zGsGVCL%I&V=g+l8%r%4HkdPA zopL~;>GZQ~<~E55P{(BudK}Y&s3mQrPG`GH4f5mXLyaDd1TlQFU0j_zh9i}yVr+WN zx_07ExnPpNVqcmlke_nKCun&dNcwzIDjP@hRWNk%d|{(ea(m4VH=o32w=S&_8mcK1 zBM0x1f-&9os!xxs3iZcZc0^mV;fi9#i9LeLVba02eiQHq{Q8UO0ZehN%L6YWw5*M3 zXOaoW8C$Q;_Jt&t@F$L5XNzx3erEhNyu`VHNGhAvZn z)}EhzkjVNC-oI+!1IIs3(2^_S)S4@X8QzIcVSteR3mUL`hxwJFH>745U8z^T?|}0Y z3y_5TQ8BClXxxv#BgJ8Mx?J4-wvTHrQ5FxRHkG3In>d9E5<@WVvretT!MW%)EwI)J zIFbUU7p$<%lj*SP+>mbls}xQZoL&~eBCh==+Cf1h%AK{uE{kFl^Gi)2LNPecwNM$k zlATB!$ogDrglbo3rAcG)METS-!_6?XVGWjv-JeP-tcJs4Q8@wOcWeHVjU)^enqL z&h-~^uDzNM8AV2)RdCpF;2`We=)dLmr$v6QLQ z+aTh)|6FOZu=7ewv?ul$L0x5tM9m%zx;GX9Dl8f6`(>iLpNp?KHV8ltj4_@DE;cL# z?F@zR7zm>5N!!~^b7#+4lw=rNFn1|e!~Uk>hG={~2&dFc*1zJuq+8`&S(q>_wYrR% z(Km-#q`vTb{=PjX3g6IpW<*n<*gQ%-``n;(*2i zvLO^yq8M*y2>I?|y)-knnvhDRYwvdQ$3^E) z)U=wS7up#Z+0-N7wgz3GTPf#8I}#RY*_RZ{{&tlvS!NXlMQj>%Iux#`OqO9O)0*em zQIBpgv&-kA47tVm4s>903j9d7%jYEm_7cl@OgJgZByVc7x}dv2$s3Ct`$gAP?Uu0F z#L^%1yUF&nzDZNLDHvO0Jxt#UgGfUkE+Qo;B`n&g&`CS793`bpr%q81nlurQLE zD>LANE&EegHtISi&-hmTNXE&%HUpoeKtPlGO&YB6dQIaQCocPF>Yc1nd$%GNRuA*Q zDES*v=t_bS48f5TQEl5GrERVKc4}+Qk#J^fZE(&(fV`c6q&TE6udJFNG`6CS1%Ia1WG{sjzN>N)=U|_uGLm?cqDA91AOK@dEaP<&F^H zV{)#@IiF4u9sdrz*%IQBwLBMMP13Ojj4)RepBl90WWZTUhwzErqCUUH7z9=)C12Qb zgbV}O%n3q<1q#Bev3X?<_3tP&UDDP`O$shWeW4nBC3dxKO=d6_lsW_h04g|84Z9j= z>_VNsoI{~7U_YNDf*q8MU~JQ=I!lz)D~#1L^F@>p!_ltzXsEdddPOsCWc@e#dU#X= z^iF#`aC>;}W}wMB3~1UuTV3#>rX_pLwS+u&!k(4~!kS%s#kVzbi8Weii5g%31H>x#kv>H(66-zfzAeDsnqVDC zQxM_9l@$L5X!R7h6kqK+UN?Y8H)186OFUGUBi7roSDr_FvhFArkWlG6W|(SgWVM8C;soXY1Lm0;@=8h$i@FfwZjp_CNiKv zKz-0aK)gu<5!6X}yXZ+ep)dfvbqp#h^xJUsb8v4VFuL;J&I!T`B+x%PliHY*3>)?% z8h6yb;IeGxWTOC_ya}w+D-5dC#*z59Q(f~vzK=5-85@s}uW5BaD4sM0;0qKu#wgmz zDM%87LBQJwSjX+OY(pqq;#@aQ(b9MOylYix<68jxr9Zaw^di-MUm%rtHV# z%#L^nBj*zY+5|4_DG2~3woC2NWO&vdv&zm<=81XpIy;SGHt0(+2WvfldWpxz)F^XV zkrn3&X{q@PFK`a0@}!vT_I4P)59dxZh2lwJnKV;MKc6jeFiHJJ+{cI1XKF42X4`Bj z46ZSpHkOfR3X0HnEEKo^?OA3?>>f+g6l%5NxW2(sgY6W|Yc9ZP2%;TcTu=HS+gLPa zljq?4i7PuCX{>0j4Q^&{4So`0&WftO?kKvDZ7#~j!hkNiCxBhdAO_l;5-COfU432~ zNhH(gF3-9;HQhGuXsfXRFQy`s#Ob53VB2H15H+cvqmVDDNvlK=3T})_2GW`;I$y6; zOl3x*qEa*j9~q#?eTtS;P}Xfytg)6--2rIT9d3~HyOgwah`VIBP>em1D_cFxsj9eL ziF_G)_m4ZimFh2ncWE_*x$Ixg{Hd;qOc zwv*;5-41fH(LMP|d=k7#r?bTnw?>ub@JrJA+*JpYHb1{y(`I_mH3o`z0h!FrkFuN@ z%0ShdMM;mlJO46+=;$YL(3a3qZv`_bs|mw3#vSkUYAPYbo3 zgJ+mjA53Q|rS#nZc2$Qwt0oM%?rv6d76HU=Q^>J^EF)98UBB+>c-@CeusQ0XNpruQ zRE4)rnN$<o~j!Y%=1o zqEG5e-yDEun}#35cWFWap!zAZ?C8$)|Digh4e1=Fc`w%7u=4I>^njj-ku8>fuClTC8z+#rigWZfoqa*fdBA4Qb;jsz7|L9tP{>A92%34`60xi*bMud zX|JRE;p^4n?J$+rJ-s9M`5QAo8iYU1%3I_dS&?5%8Ix_r!t|W@8rTYF20Q8c7i*M3 z!a9YT2xCl-95_FVkHaI%0IjE5I6?DQv>=71)W~g@V^EWPV{k*FkunV!Gl!eryq(B%fu{U$| zhT`%eXYn+DWr4aTz7+*Y1rB}V0@y*!M_Km+wT zN@x&Qw0K}n+IcBZ(v4JELtMA~f-g_S&$n!LSw}w5JKHyKX8P}Y$p2O-oX&)=@1*Kz zX@K&UI?_)pJy$Fp$`J(0<|5BB*c~{iR1hOIbXssR7gr6ccG9kCK|4Zga>5Bk-Mqw^ zPBH6seDe8;0k2<}fHdz8DYxMmM25Wi$Eohi)yDVBQtryn`LKwnc(~n27i& z)FK`?HcIW}_+b!Hs$rHVKiG%BbWpMqN%5mEa ztGTb1qtFEo)oZD;m?pJTWl2T5CR3aJ(rsmZLIeK0)|%awR$^15N_<7%STpZM9iTRZ zeA|3}V%~~XtmYzN!9@y|HRG+-c2jYJBQ@xyqGF>{sfPZUs)*uJTiSn1H(J;0 z>OGsM(Aipgm$=}xt}Mk#QC(=Q6enM!&$tiWHOpdpQ>!8=zigD)n{|DPAoDmNm~WkL zkbAy->%VivRSQXtJ8GO*K@LW?2H3sEeyqQ@NAXoci!khI%p6F^PV5 zj1ghZl$I#Xh6m$aFJ_=`({jZzgBNj)ImHosk;spne8RNeL64QeNUxiRd)`;<0}Ggp zdwO>|T;cpdEs`+L6}*rf0GEYQHDDQzbFa5m%XHhRci>xjQYX;0%!qDw29Rly_Jn8s zPNk?1sc@h$l!rscVeg~2bf$wiXCunRfJfF4Erw}$Hpblk1uOSI)l z{+9$kUG`VY66vG-nhENdn(_Ks70L5wxvARHlQ?N+Zqb-qMcgaj(TDZsigsjV02LfQ zNToIUj(Ph!!qQA~n#Yz{7yya|WWZDqs+Bu@&JeO|@5T6pJvsL!aEGz1KqEnsyKK3- zN>rv4au#xo)lIQe!h0M2=jV>1=89TW!q9mGw_WJ>93m%-q z;tj_2HydrkTio5qVFdqMUs(^L!-A)Fb^{xPCWKJx$QBuH3G3n*7hof_sZd**CG^#? z!R(pQ5pKTvMucVr)fN#ntATLeCw5>M){)R(gorjqs?TH>J?{hIz;C>S%Pxp{Q6u-- zq#R9v{yUnL9MJfNSHKVVw=>R>nwz3$+50F1HU9SpLI;=*N%|VoY^_NkJ?VgYyot>+ zMlJ6=V^=;xe$_u700?1JVQar573ZnKK{s;8vXe2($XQK7FTQd*tM@8mO0W8jZ5Yn=2mmtnkRAEh594&`dr{YO7D5@LZ<4@vW2p!)Q&7l7W+vYW*oE=5UvV_)92(pU z-W|eH2w>%X%Z++kc&G%4*&K?T&TezRZu`hv^nJg5f%h}GKRPN-n20lS5L}!j1i9tx zIx4y-del$EZzTmpzhY0vTp$Onz2qM&xB8UMnv)2CH+5C|yuE0jHOHocC%IYdqj=HZ&~xuHDy z77t&4*k-6wdtav-dZY$EMN@8rNmo!AOH$QyE!#?>bdQ~8Mlkp=x9+ea`&2s8#{xkg zo;UvfG^MSp2*JT>P#bGz-Czv)zQ`jCtr9_{%ztA8FzP&6KSHKs7vQRv&L-f)dYLK< z9cBXy{0fs~DC08B&n3+N+DKK_7Df1e=M zr_DX@@|0C)Fr}~!G9|Fm9mC_+9*|YHG_#njbcrP=$&+sv=LoU3w})&EHzmtI;WN+* zl>K-ZH2x@@%@%8xuk;P>7Vm*Op8}ATB)8Co5a=w(6sU1BK(DMJC#enjs$zUJ3w%(k zpnn8b6W@S16W`$*yQK9}9W{ocSOtOq8)_C$WCd`LKtP_zNq6OxNqb2IfJ+bb74*M3 z$A4yLO#6|@QWTA7f`hX#p&7l{*hQ1asv&X9U>@WQsr$dC0S9Khn8^nls8AZ0y@97$Thg*;& zkV=x6#a4B62A@i9dK9EVgUpQh%^X;0NHm*9DrjME6~$Q6Q?E(34#SO=^gQUB^e=BG z%SEQiZtm^_DA`@|0pwgByj5F<>7iQ;j+OqQ<>(TXTuiLIUyT9*ie)MJOFR8^{1pXy zoqlnxv{K2H>F8^TewlW69>3r^f*^#{dPbVX<@T!hp4Kot^6ROHDRs=l_~UUF5J;DD|rI6hOzdxYTzNuvSu3QZQCUFP7`TC08^J1wk|KfZ|qmIucV|e z({@hrW68pjc#2g2l;D}wo*-PjtmdmG8!={|&rzSVl5YyBOM2=L3PA&q-dx|_?qwQ1H)2u!DU)0;hU_tX2+_c)V`=c+-pdiqCV%Jffjq1Vt00PqkNyDYMXzDwuHg`gg6dB&% zg(VPb1HHh{uO68LG~?XxV=Ie7dCMtWDeZ9y+;Sw;9q}m9x+9ZN%H zDqb=#ms3@GJ(HxUS&#)Su&2eIrC_+WDNO;$0|M}u?19%@Xk9$I_;mmvoU53kWx`Qo zZev=DozC2PM>(u^GT?6V0$<6pg|9dPS+ix3Fkt1LU0I8_b*paY^v76;O>%+7jGn^k zB@5$zqN}FHnZ3cz#<Ptf=W_4)kmp1Blqee>VZ!FO%{9w+ z0QaCJC;8kL{Ty)f6J*KZ)v)RWS+P}pT3f5`kNic(G^%eZ8WD64DQz{Uq9^l7Q6V{UBTd^-{fQEY)Mo=^!_tk)qJzIC2 z@LODP)J)+}D5?cs7hB$NFKS-o6o$_K_JmLL%;5uo%8Y0=M{zEY?>@*YUhB*OvhyK5 z_T?Y!SZhS6w3_2r_1bCg+{;e+6=hzEowhx(9ycDq0)mFp-Jn;#qaP4wPk5QL;usn7 zwWEbf{5Y!LB0+|-`1K1=67&TFS~xN@s8KX|vm)fu&kw)x#1eib?6-w7GQOa+5*PXn ztP_aa#8$Ns#B-*Y1zaJ%tN%F#{F)3Mx-A^t(?WVg|B5HDIBMU}xq{1)4r*Y+&lL+9 zVil5*@HhhVA4Zri8leXvG;&f9S>j5~et)4gkh=yOp_=kg$*ijl!zWq&VBPbVntGt? zwhIOx9!sYc7@)aVu*FC-3I`ngenaGM5g3uWH^|2LJ7ji`Er9xW?DWtDfY?D*;F8SJ z9!8=cITd`MKB+ak%*bNoh2U{iCL}=j%X0FgALEJ!5fTb1kg!E`P-`DiX=VYkrvp&c+dEZ~* zc5dNn07;xwmu=yPCQ~Df_H7Ok6Zia+K-~blz}15p zjnQHV(8Arz<;2`1~ADU=R*;6FAAknyT0uvmu6IZIIdkm;kCjc zqtmLU?R4SBP>C*+9tK7rVT8esPU zU%#=VJY|(n%6{q{K9nkrJ!P*T3oa;@>-YLu;2Fqu-6yp~HIK2N>P5frt;bZ4M|u$O zVyq>r7geRI4?`cp7~arjFU9Vb#pqt-_-AA1G?OgeNQjVu3y$-YuW1E6HoqxrO{S3{ zl$BeD)vqyV&_^uR`rl%pr%nU?^nmEGhGC|?JrrhH{IWi{rZER1kya00EQ&a$?ED4j zsT~8NoxGP;yGo{$?S?k7SDl6OtU;6UCui}{U41Ff4butMZLq)n6!S(Pg$}p5Bey91n|F92p24?6UhVbqNzQ};Mhql=*z5P^%)IAjec z$>-7o`g^br$1UfXX*|~0y`!zhNy-NL+asajX-ZX(*@u~m3iFLqiS^ww_>NK_ydWawQ#dDeH5Xq1Igzqy#LuQ%O?WHPhN&65vQM=W6aXFO36x!dP$McG-w}gH-=M8$-wD)IJ%dSp3n&+*G-wa- z3i8;VgLO!-d(w&BTQSV+`vta7E0rDZ>9%heEAle(%0HRpF{*3l)D`8V0mZRP=}QZM zI29gyW6hE7SRGBphH?|=DL9vajOy+?*#>jyHQE*$|8%4JG@Z&F^K?FfQ~#dw0Piwt zZp)WH3hOGN5rB%C$SydO0hinit2w7-XT;8I1RUQBoVpc)?t3uCNAanzDbtj$iq7SV zgKlkhJx*3mxi-%ytXF0Et5rZp-&hS`#yf3uqC-5;Njg&?jmJ+?1S|Owo-E1vZ~`?{ zScDBIl+_d+8o89+D*84M`-)b0beLf<@2e@l3C+S}UK0a4bqMKQ8Lf!5ITv!1w*r}D z78dIz19Kv^Je>Ild@@X=7iwgHg4dHIj1O6Nm^_kISA6qyL&yMEjY)u1R@D#C+~^SA zEnpqx%|w#Y2xjD@`M3g1m(7Gk3Fb`*qV+ zwfFbunG;B+o$?429ko6GupMi(f$pd<5>8z;xq;@06x@ae&6myupf`dHtXxkVDr2mr z%-mpA^VDlRt)A6lR`)ey)Y+H7yjEQ#J`#)4O)6{~f7bCbSH z-)oXYq2ZFXy?dBTzS^J+bE6|5;@*U+S*zmY!MlAFaLH_`Hpbi&IpeoSHr$bU*&<^X z*Jj(a++2fVzlg@45QY))QoYN91Aso_ECYK(sSa z?dYFjqWif8_`KxRiZ&Z!$sB^~WejAK>Nn32!3bmiO}D=~(j0;d=Q{e#+2R0?uD9wG zTUO~wZO)TjX^g-F-~JLK;%7RSw8s}WOd zBQVU1zhuRXmgM_VVf+zafYSc=&Q4arRZsFR9jPc10Pfuv7i5&!i;ALr_%MHgubm>v zl9;iL4hl9Jg)ahx-P3r%wLx z&9fGx`4j4+2pF!QHA#aM8l?=6XRbHuh7FOju%OQGOC)OuV4aaS!TbA#8Kauw5jRFh9*T_Ram_KPDtx+)SW~ zp&RZ(<$aGgr4dL|edstQ?XSXNaZQJ9&Kosm4L2Cyv+b2hwk|ZkrjInSc^g@hm9m*CqL_1sSiy4|Ri-cG7m|sJg>RtUM67e8 zBdYS5hCB5Q#Ov(-0pxX@JGL3shV))@{J~Dtr)}m>O9tO+84?Sf5Vp`QD{G%cs`GY> zDsb3vuSv=IQ+m>Olod-(HlLYz#eav36$^CT+18`8zp$#!^6ZcsY*8hA8 zZ6m`+kHbo}o;Bv+LcJ`UBIhfEt3Qv;(~o}->&F5lrxFV*;vdu(C4^7{Hl!{>;HRCVn@!1Y%6JsJ;_9<6_vCJAw!_8FMqb3l{m(R7#C#lVkP_%`FCP~DP{n50a{d7abc?%Rco)P~W z;IVbD52;C)<1ymt4e%7w*!N!nD3AN_mngu1}%D6Zad0YKTozkRPw>+P;FMD5H3X_A28{r*Rh!G<(Jkjc`E0?Tf1l33m)eXH&!4@>sJ|NZ$&Jz9Vdf)0($#QUXK z=yqQ|_Q$3B61tJhMg2V`N6ozZbl#zz>s+r?$y4Xr1MaGcPHgJAbsG^PHd8!HeKb0J z99FBhb9H!sWoN(%gu7*)<_@QEBc!l$ZQbJl^-lkHil+%r#*n70QH7Q|Uf<-W3JBm_0f1WRK?2kf24u_N3u+lJ-X&UfJXH zCU_p@<`~@D|jSzpvcLNn+I-;-00^U=}m5( z*jYRYdL6}3=)Zw}DUCk1+BOOWvRTbZJE!UeF zjS1((d!0)$dW%V};EsVcp~C46TCXMD9E`NzR>l?Hx|9Nz;UNYEF(#Mp$*jc7^d-mr zkZ4@h7*UWZA8$)s)y6Q5&K{=JITcAQiihVtasqC-a}4<~oXdI1+*&FgI0~3B3_chn zXn1(yx__ZeKHp@#k>MGPCoDxJQ%YQqLK9zxq{SyIlGZpwNN7SIqh*Ca6r{hcx#mq< zv6NjOI++97wL}==!vea|KFxReWNxYBRpQ&6EsfIZRSc#dyKCY$-@c+1KMC0eZoo0E z_j@0oD(YqowY@X>=uDy2zunf?9oXsF9@j+{1_g{9GB!M$0bAS`c^yIFj`d%g&?_TD zdN!$e;B!nE3;U{oo)j@S^AJD{+>_bBJT1@xh6DgX$2e>E8~6y97sA%x+%<7>ydh5k z$W{Y5DJ35ZHD#TG%&wM&&8;c<1L5%!KJz)W-GgfY%W1!T2HDhEn+NiL8Jv(_cyMA+{#o{EfPg6e zX;Y*C1`}H&7nd9@D?^=?lz%ez`NSM!7AhsODY91hwI_5y`mk0GIMc9vKkM`$Lq3}qOze3ijwv-AuWnJHPLMPT97_Zz=N zx(yhP?*fn8uGegrIsPe+IX~~C^+!k`*|cwfm*r5|#eIhNS6zfR9?P8usy!adJ>YHo z_zn}Im(7uj5b{0+8dXB}_@F$%gaYEov-cMnp%0k^zcaj3#yW$2ou@1u<`7a{gi_)u zx=PP!RaglXM8;Y2fIfDicxdGq6T9+$%E7XjxM#_Hd;$f<5t2r#QfM@XQm}~reM1T$ zY5B->gjt+_lm_@HdHeq2K*QU*SLNZpQC1O=*|ac1I>U6_6qWTvRiqS4+0?idEvq~c zMT~7FUK#b#3Y?l%sm7e9WIG-PH7cba(KJw_bb-b>Gc)5RWod=R!qY6&npTcm#k_Kt zPNedoLD$^JESA_s%_CRatycqsYNZO$)2P_P$YR5;3EN&~u*sZ2Q~tHp5Ya{@vmk|x ztpYD%glh{S%^I3Le)z>^E^&Dj&~7)5Q|q}4{bv;>)C^mnlBybh)L3^jqb9cXjQXae zliOY*XCa*!6IT2zp($`dv%A*>^m!8vic_gyxQiR5#U}>0QNZl?#Y{kXPQwj(;8%12 z5Tc(;`$sc>#UJAu6b$KQvyB((@14(7+p?kiF8q zbbyUg@r3$^OG#&}9PZT_j99rXvK3sgIz*w+b1s(ah)z&4-+|4tTeg_l&UGavSCggO z-Z#c;YW_y%;<2KzzsVecwk=64x?i`kWI;YuzsS&B2}ce7y9SML-8ogA%SdYg``g}G zIghUZEe_w_*nXaw9WGFkMfsfo4Quw(iY$sObpoF15P74ybZE=xGGpXztU)@fE?}Q$u#cpbUPq$G_%&#)6HDR;} zH>;RA*&)&ly|pbDKpz;9tr+Hw~Q znq~hFS_!l^8g$5&I6lyZF%N8ZC-mLq);Z-tBRnp(s8ALlH@d`k(Fgv6%)G1;WiB-K zkNHXVf**!cOeMxRU$qa|EVWWZ?)l^w>-7(+mx;QBRbkd=v)B!?3?bd`jt;>N90 zQVWvmw(6O(Od(1W83TT*E=5)ZRZot^1O~C=adL{5bB8W5)uqSC6OI~vb4tCAdh4IV zj>}SQ=Mqn{Sf+9=*YAzw>w(em-;^nco*b}{seLOC7oVUJQm6k80 zM}whMbV>~^qtSL5utp=24oZx8Vd{2qGQkQbeP97a3iF`f4qcn3bJe!_#PuZ7Ug~w~ zHnE4cbK17$#7_HK;_Wzp>UNdr^NUWNX6td+1OKX`mZ9><=p5@;CyDjeQ`LSmu=+24 zPDC@nCjLdkQsU?F&oKq+I<|dN7D~WaHD3PR?km0wK|CaDt1^BJ1P?k3E1C2+#t(E~ z(=l;}9FPn_>Mg9cD>Fgl`%z76o?So3W`tBf#Mk{cB!L%Khac^`AI5+whGv z_q%#!(f~OQJM>3L;~z<`8lZUOFL+t2hX27!wAia_749mGME|S}NjP11=&rfsD17-t3 zwZQCaLWYMpZxf*CV?#83EgBT_Gpg=guEZP8aG>3(C0O!hqcQ}|Dw5k&EJMvwYI*%t zW3yFPj-YgtVx!#C_KtD9AdRs!1)T1jT_ocCdh|Cffst4hoBJQx>j<*BAXfx@g1piQPhm zI|lm-@s1{n&sAA?As%>e*rtv@S+zNNTuH{K6*je$`K-xj32HCUkphwTqofs6?n>$_ zkZtQus|x%VS@PHDjp*85!tU+lyOl%95aV&dmm~+ppkdaN#~i6t-_8IXwC3zg`jOcdIqzpP;gOBxpClgSgq~ldUN7g|h?rFfA-xb<=>!y<@tqrp zQ<4U!yb+CS#=NUo@DzfrEmaCItWCCFzs2vGH*#wY>5th#mm zOCJB(i)&HwXEma7qr3r~VV}UqPmH)GjpX(UbRAhD_zkPa0}E50e2|b>nGrT0)eglr z=F{fjg5K-PBSzB^#mzdxqir8(wqnoZof7jT7{1S)JPysQw#W5rJhDwqlC^S^xQP^U z--sgPmW?I@zFA?qFI;+!n6-)a?YJ+4k9f|x ztJ!-;OQx{XzhTwZ2Xas5O(<_nx?DURj7#xW?Wk_m==kp zp%i-{?~zUk&G!H!FP~pKaeAuJ!Y+G(xbJ937vlONoYSF+{No${E*I%*`Y@Nb;s)VL zOPeu4K6SvtbTK(YhPs=m1{ve) zwdoM-*Vn;PUiV8CKXpj?I%AHidimxuM3WDwqeL%(od^J4=R5SU49=aXV5JT;nnUOA z?q}R0{_uy$S|nKon)8S)b#6l|M?QD(Q+M9a86L``C3>S&c#!nrCY?vB7?8n6RPtGG z3C#J+P81+Z&}iijM{4tSr^V};SxS=%b<|YjilM3MDlUdpM0#3dN~`gfPF_B`7uZ6= z`0gINSHJ-5)rU^pZYH}2`OF$4_0lKw?&Yqh49v_U*-%MllE)u9FCR=_?={>_1l$r_ z-B2I9m^%b}&=M;2IBM))8?4OqxN4rL@rh}2bA_n$p0Q%k)5BqvVp6y;tKQs^K`ad$ ztHmi-_pPhVtgB7>4WcFL349!(^(jW`8i4ymNCg0*r~+XzN0zBVmI+FhDGgAOgt#SG z#N4P^#RNuLLE*N;C5y4;x)JHy(d~SZ2ws7=a|J5?p|r>Bwkd5+F<;PI$11vvVU9t* zAz8;*_7h6bmBxRe;fnMLzJLUe9|@w%)Z-dop+e7?gEF;d>D-*o;A%O5pI~Jpj0F5e zWBCBqEkbkN9rpT#8qs}YCdI7fK|_}(vjRg$c#rwoxnpq~E>*VD=7c_o%CxJ;4;Fk8 zc}$#?lM-*#xyC>IIkFyd&u~6-&ZY>V1VgHzUJ$Lg>n>odaB#1A%x25TXbm1{?7lc> ze8MpZCc8%BM`>K-r%ZfTrDh$Vs&fr{Bf0?heM<-lG-WLE+S7QQ2USM2&Y;P?j8I{5 zV_f0rDH`FA@PAVJ%?yb#?u$_$sqpr8*FZxHjKhnu_w{mTBWOzPwzNk<*L+JaqiEu#y@S(ODQANd=U^l5^_UEJ zl~-d7r?yB>zOH~fZVSdgt0^VFy#xV&7BF_w6}IThol~q_RFS_>AiPwbQ84bgj4M;p zL(qzdlNB4Ns7;we=!nU%Vw3p{^o1v-S7Z#kD(1@upP4A$x;Of*1ZNBDcd+(Y_0~N; z-iibq1~xt&5i|;5*rl&*#@8A>2C2BxKp6~j=eFY>ozcJ9@~ZOd4J#fe&lCY=ay6TM zi_1kMA~rno&nTHVA@<;nQr{&cE_UY70S7+fRqipn_1e4fk zOcfS?nW#$JF65xNCU57)Yd=s{w?TZ<%z(FUF!McV(3H-<0Teue9mC5*Tp3W_XeMmE z2y^`Mq#ZsDAlaBX{nN()p?|lfy1tmqsh-$ZD6d63+{0_at~+)#XX>?7E;Biw-8^At z^|TuaZI&*nWaXEY)NUGh6Er5{wc9FHb7nBB^l!R%#sE`T+IvA(Copt(0^hERb=>Jy z2uD^rkpPNbKT+RL4Z8k5BjTU&Z$S27o3eUX-x%RC_Q>Oxh_VRaRba8CHtp%J@c-iL z9fLCozi#iD2`08}+twZ1wr#6}iEZ1)WMXq-+sRC9Oq{&=Kj+kQP;Z^8uIlRQ?r+_@ zu4}LLTU-7Y;*g)lGj_@G$8U8lUp8F{=80Zw?r$L6K=G=NL~pLLsgc2#k?Tw7PQ*En zF_iD6OQ{6uX*$O=t}&Y-lO-Xw$7=TQHy5JZe@}6Zq&AWoy zSm$5&v|4|_fIcYSvH8EO6UB9ydv(<@0urpu{TW`^B67Tx(Fua|*w zgx#w#vlN1a2)S(E>YO(;7$oRyKPHD z*HTLGxosWk+r4d1Nx}`0|MBPx@L!#;pG^aBP@gbg-#97QC(M`s2m9yyRQrGMzP6dU z&(mja1uoM6E<~F#y181}GJ^d#HV9z<|J=5+cQtdiH~R1A{u@{#oFb9!_!PJ!AV1N> z|JRF(My{5s&PMKL&Mro_AWJh78#Cws(P>Lj)zQFJLwgSeH-`c-ssIfTG~(WYvg_F1 z!ZHO->B=>w)Eq)3VtQeYnc40vsMFa2d{29ML_lwvKsh3{Oxp8BoO9bWS&3B)B85%P z_}0tW<$BB0>7~KP7e|O~)0wz&?CAZ-h$@WQjP%JJkZ-au%<{d}R6l&zw2tPCJ10He z$qT(-o1M{U|HyN03*b2NDR^{X2#oHiY~T)BJnB}#U|eKT;b3VZ6(~kQD$lpMfgSrc zO~T{RhsR^${-S5lSXCE7DML!ps-G&Jd5C4IsU+dp!6ebTrcJc_zTrTH{79mz8%lSj zkzQFV-8^765#*bamgmCl&>Op|QtKwBPAYg<20Oiia0`Z^4!kBNl2lI|zSEQRGBtR3 zABW0BrsQo~+#|G^km$p14`LsO+^+O`a>Jmb8u>~{>Z>Pg&^F>L8A*#41PUVyqU^di z;ZkamyvR~p|7Pi-Wt+0B!{UztwTHzhM&KA=HV`Sx7-3Zm(zfATPT^F+_F#n!i6hZ1 z4t2)Ndrl9t0nU&V1}fc{;GO+Jp5>`Mw71?_MVxL7xnq%BGGDY(7mX}3=ro29 zrV~px#>H85C=0c6gyR}-*G_|LEYO-FzH_Kwg@pQB7X65VgxemjJTSCfb4(h5$F-wu7^>YF6#^m~QmPeQ~&mAPQna1Q_3L@Mpb0b2IC6%e# z`=#_Ovz4-e4&*_A-m?PWu+2hWkcw_D>q<;VF{96Q_a>O!e6BxJm zUzCt2y)8;dghQ)5Y1^22aL~`0z^7bEt0Juy2%stgww858q%=Zm8Cx2m=!XO6ZtIjI zti(TN-+-bQrg=e<9B-#^0db|TtiN|yF^Jcm9jWrA$f2-`uoEh)0*(`d5fEecj)sV}uE*)CZ9-{kiw5;V!u zbpTPzT|uY}NyHp0)=v_ZKI&qleQ6wZK`ON|RgN(yCs6m<1<2`at7uM+s)}O3Cg8qJQkcRLI&cBVeM3}?CF`_k_9Ji0Ya~EIF zgeQhHhLDFA0Efy_Z$ZRbG|Sk^xWXz-(M!cBIynQO;aj(%3c0@YSFHwz32g4u8FMd< zlULOkqV#3x*QJcvv8mmpDHsWR-M7ShMpT7qjWwjJM0AS02|dW=q@QF%iqY69&aw!W zy38*y|3Un`s?>dyQticB3CJm%<0$Jk8=8WgoO8}WfPJk~KJgbh_~wsWl8pM*H=;bxBmvv!{*87mQkT1?r!!5@9$TUE}R)sBBIhO0q5454^ z>8#*==_v|-kM^C=_0j?(vHNjMJIuHxW#|iH(tL7q^S9+uP*_!Y{eTg1s#EHGA#>ysf7u>m&Sx za)BupYnBhX`AgP$P@Z94mEg6*Yvr8t*Fviv&hgCi;kTmZlyfI6xs?2zb0L7x;~171 zBC8f!BvHBSUVB=BBto;| z+oO^xRR;{SIk=+kpKEYF=MTvl*0lBi7?Y)=@ib^YNTI>se63N{NFbW0P@NSUSMC#} z8as(jHQSVOpIZryzjqM(R>J_cQ`#ry&en+ujhn$!-&tznx{wgB6!ZOdhapaYd$XVB6P3#zBYrDBvz5;aku#Rb`HMdIv6nyWL z3~_lj5n;DRt`Yj8w3cbS+eF=}(_b%BXjc2owszr!I>cor%cn;rM3<$=u2}ep#{mmvPW7Q|GtLN?f=fkMr z$J;wrFZC-KX+UvwQY)1aN4N|M!+82YCmbOLStmU}OimgthOObKHq72hno5o>Tx_O0 z0^dOQ-AP4q3!a2@H}wZIc8+GkHNlp{h@CId?f{YMuB95CR>L1MM{GY5Z!8;>8@RDf zBS+ebs^&?%&?T$MnQUxG1GeUg9feHpDMzLf>{scE`mvhRlS=tS#j%OX=SFL*y$+H) z-cualDZE6MHt)BNwftd(PpUSZGp$n}8~%DhoR}KxT&y+sMy4^;%3g{QQ+r~2k$pL| z9{Wv1lZlnM|5%cx`Cek_Ej4DG^X4#Fc3a3l;pwhA!7jNrB>5nN*h;b5s!p`L4aYYGR1I41a;}AN{1E! z=#Qu?a6hGf79Wy?9*eq?4D2FXa6k{u*nX*Dt(V>S%OxPv%@yyJcq(mK*&L2rRdwstB5PGU&&I=^oTT=~lmjhpcj>c~v<@ zej~YSf$1IqgV^5x%DxEuZ(-+05aAW+z!j{0zWo|VOQi%9gD*Z{G}#10<951w^!{FD zCsl{Hn53~?B9V25UW4yAfh%*1#)(s9H~3WX>^!MKr}9Fl((eVfyBOqg(3Z3XG~(-* zJ3LudH=O>{&TZCJp|@A;Za8tR6!#uuNRWVcSYD>DofA6xol3V`{=BJ$xs(KR#o>`C z2dScPhrPSz>Rl&WT427wk)clV>!hCNAzRto$9NjYYExl;Hexm!{+7Xki#cWgIvZ65 zdA7HvSwfl9?joj7NWGI4jt#B@IIyTnahQ9p-_$wB0CkuzN4Id~xElX#Z9mz>wib@& zcTyX|DJ9d}raUIV1O%9vpDpnV>%jT3`N2rn4SVKfVD(FXEux1q%U11S6GdY2V~aCM zt2brbg9?Fsv~A(>l!l(!E?5JKKMx;)Lc^E}m-%kEb9o(5=8_Ak*D)yoaoVgrOi6UC z9bWp~<&NQs!EiynplfUNj42Tw6h86kR5YV2o8EB4=#nGtEnn=M$MR(T2d675RJ}ju z8I-AZ9O@vdC5Z9O_-F);L{fQ*x>Hn;5z^8Up`Un$q<|aIm7Mq*VcL(k*24k>@v# z+dzqI@`3t772ys;K!Fmf`x})(GQs@ES*YU|R?jHqLPvXNq8DU>Y!}!M@H4Bg%@Bv1 z&jv(eLdkRIyK})G|HCeylnAx=zu6xMtLz@=Ki6q1pUkB2f0fW~DCne0Xi8vP+Rdv{FgyTvB{i2LLi|8$Lar#h`R zdmiuH{YL6Y3w?{eP42FK2Co1ZEz&#tc*83$oQJXnk|*WwS@clDDWC>vxw-J~Rmc6L z+Xa2rDaZL>l>KUxhU_%T2X2-yFi^=1H*phYzWGlfYQhi=FVm%2tIEl_)gPm7_aj`q zLP{N{Inm+yUfX`awX(B+CV@tPrDZaD;BDfqM8w@+-*Rq3S`6GUyxYn=JXtR+=vnMS z^$mXPj4#+9OII`+2@rX@4r*Ie%tq7A8Bk7F(|5k5fL2xcu9@?<{V?J0c_#EBnCJK; z8pdI6@>8lq8$!cT;Y8{whuf%*Jqtrh$D)Vdyg)HFZ=q*vN!kX_(;xai7H2FAkuNw$ zL;PSlTo(DTn~+mQBfaRdNLA4AdAOP)qrFfJ*moPg`9`a~en7RQCVDC(RSo5gITxdc zowT2M#l_N$nIe<_-!WB4~a#oJLIjsXHxyrv*<364=WY5Qo)5v z9)@>0LghHqswG*3`1#+IM-R7)WH0x-^xafGdK}Uh$gadoE%_82hxMjY-)mwu0X41* zPsf046iNuvSfFB~f=h3tg5JUNB9;xxYL;{$h&oebl`SuvfCfzzkva<_rRFd?+bG_C z06d%#ZBT4v<{lGM4&)+q2R|v?_)r)(?El;%{{k}^aYvx(ZHpOr_sEdk)EKoRbIdMe zLa7=6fow{ntqG-6S`Zupjd+*zj@4h^X^)DMHku-Y0L)~1{|tOU6}f{jht$b^3;1`( zsQ4b5OK=8vAzX63>LO~%iiG$EW2ct>AtEblV(J^>2o;nlRv%e&!{y**qaA1iF)V zRMJCd0T3(*#Sjh5SjJglIkWq=(qbSl7&3?{GY2cj99gx+|SmK|7U4o`|r{Mv4jPr zsJ%F#N+7-evvvMwCKrt>se&UJ1>2mof>45pqTffIyyO+HA%#knbvwrG-Q_ZU82So& z%TV8G603ZSkbcvkA$}IBQH6Hia;a8XT6QC(#k||3JO&!{lyE3_AU?~63|w$U z?C(1iQ0q=A8AxH)`H$;)75g!LID3{lir6yRzr-uL5>AW(8t}EO`;C_@RCWN+%E4Ib?J=tCr9TlWhydB-KRL zL55EGM8g#d`su>jl8P&1uMncanAx8USZ;}Vp(oWSOx6u*oV0XKZuUYsE zJDgYOU?W_2d_oZO-$VT4uE zB6*(V=B5%#XKhVG353UOiSO{Oa3C$uNZ_t;xa zaiHyfl8P#Xu5>S`^Y5Yrr0ty$Cgrgg;AI7|D{)-ayU^aHp+KIoEum&7ItTv^Y&(iX zDIsYCAD{Awjn0vuP#=LG5?OH&znlTu4tB_6wg6GjJm(Q&_DG&)$(PGL2-(*dn0NTZ zQN`Y>qaj{Xbdo##Y~qN7NiJnFY7y%g)%iOBd`%{u$GaK$2pAE@Ltw{BCku|>qh)P^ zXch#5*TX|qgvjAl*foh_h;UVjF*OgaF_IsU)a;|r*kID$_mE-g5a%_XKmM;B^Bfq_ zEYBzEz()7w3-N#b%=$|Rz@#>`uj=BWAZ5mcPe!LLm?-2|2?#i?6dIWzWRxIivahVr zZ25B%H;K`*=5tz_wu@_#?mvx7qO@C@QyJ!rX;G1A*Ti%>R{dI{bkgdUE~+LU1RuIq z6Pf2OU;KOgg)TP!Zum5Ra35|#E4-feg+*9me{`kOv@Hyn>0Sue0V0{wR-HK0H}P2( zK1ag2D*bA{+R*riZF7%oxf&vnc^3uX(`j2lC_KZk>;fQj^YILStEuDzx|r#Iis+1q zea2R)P{A&)wW)eCZnd+mOCZ>`se$_qqNo^xwc$hWBU<0w;c-|0lrW6zr$*<%sqw!q zArsZf<|%A)b8I%9fc8Et|HAK|ZuSY&QN>zCx7ts(i*0fOhd$+j%T^n;x86C7sY+pC zxd-xA9KVlo*)Qf=*FiRIBOn{kq_A9c`)QaK3$LGSX_E7&1&3Tay8;(!=225R{7sR| z=n1bCL^Qh^w*g_fyLLPd4_76=8E37!<_yc8VN-RptI72E0L~>^&G!=Q#NkA9%}it~ z@e*ij-;k*e(c$EY2EyD z#J3k8?+Lmi>z1ByxcA|4463ei-TafLmXA9P4o}^>g>CQ6cb_qGwkvPfUHvvKqC`GDnQc?&JMg|w~P!e#6x3=8q$ zz{Rn#rZkK7H?(5MIxvHvWe$U7qn3Wv8LV$FwiK0h0S>37TQR)PJRR$t_Ydbs7iT9Y za7`_CCt8?>ds8BXE;Q-V+Aqs>4wswAM;lAA71qWy>l+=%dt3j63?+2F|Di&AU~U5? zBp@R{-qlLF7CVtyBZO2B^DDXdZbWjpSBQld8*P}xYRMAZHEYglGNG7PV&#>3`gObL zPsww&0VK_cZ_L#JOay4Bsq5lWgTV#E_AYp$LxDSzE76pNNIO9ri;MI0Tz;hx7Rezs zkGYbr8U~INkz{*{er`;7(Jf5B)Ti@bD)4$-VUw!pSxh6&`sx>xu-2`_UM6ygWpwed z*+4oImWsnu#k2xAKT2WESujlPU$&)}s0(vA04(UsP$#f?IzM^0RAyG#NtrUQd?=>a zivf#{H*U%_q379GB@fQWZ=SXs)$EtLabieHYiUBH3iQ!(8z??h{ z0Cn;9;Xp-x=nH=8=m=h6Zw;Y@-BPa|K4zlsSw6rL=3TQ)QH}MjV(u$PqOJ+xP?nSE zh}JwYUF|x!zyH>ySI%hk0|o*7$Evi13)9?6TZP|c{wITkPEjaqI<--x1aEv$zeor# zr*+bw_*EsQ$X6>7f!&{xMdVq?6x{DO&_H8-gcdPRQjHi&N8V$O03Y#JX(}||l<90G zpDqoJoy^VJg3DfPpcE=KTx<3hX{w57g0nkCXjq95x~;>|?Czju9l3GxAgL4Uf?0^f zUXd217N~-H>x)m>iLMSAF0adQa3@fWl;O`v<+6&ceyn#v>)f9D86%v75ARG)j zl{@rrkRjIy_3lLBaErBX)^Ll>t3b@9Brll1HWxaW%3Cv$1xY<49_NF(j2c96OJ>8) zj_Lya+6VDZd|?VKC!XrhiIy>9wvUdPIW_-b?BE%$z7tt0e;Wr67tL~apYRR<%z7Q>jxWS> zrj=gO&pRMO_%w{8hLaTrrw-E5xJI@MTf(OJ;`QTb=Lo zOsr7FCWlDC`wId>`-nKcjKJ?5(5q3fU_YT*X3F7-;nm1v5#;uW?Um87U$$VN%q5&P zmMiY2{mjczfUw8RS6wN-^5k zQ<5YH3+}k=z!s}h?)waqdEVUP*WftkxqO2eM)=$v6G+0t@5p?N2Jnh3JZsCftdHOW z|9VP4g62hoydl`Y+K`rB*aU5_r2GeWB!<4{?M|0Ysa~Cm1&MY8GGPdNCNvy6sD1H~ z*)Y2Yack@ogmI7X?|6WI=19EJ+WVqtNidny zS0~=<=c)w95#z{rSV$ltkLgiH2-YtuSJX?^&Or^LA7^Z82#K^~$&_)_${_WC77|dK zxMugC+o`*?fPV)d0hN6R>{q+@*uL$*h^$Y_7dMy8Chz$RpBNUOJv_VRD}Qj$v>?-C z*efZ;+~WwOAjHe90yv39rwF~UYjJF-+1tSE?{q>d61xsMRIF90k)6CqR)Au1TI`^# zzUeIg`;5=mJ-dvH z2*>yH%X+IEdvqC(77UVz+=d(AW41xUBmznkp^{Xs{+T3w7^B0#6Ax;zwXzbk!M{%u zkE=34j|6>57fdk6vJ@Iac2BUpk}fN6vm=Ikrth-6Vtmqo{K)+J1T)(%JSPFFqS1o< z2sIF*G6i6egIuMjI3HVHKfk%NuBbWQu``hc%!c6J-6XwOw)dRAo|fzvnKX4ioWeO?!Ku8eVcE48+ytQ;81 zhBu4gkN&gd>d;O3udLY#D(PeAyPD?WQG+;V-p9g>7Hg{HXLmyl!$Zu#{hsE}=N7gq zn=6AShu;uhD0YYKcPVatvR;pT#bl~&bJ=@f=dAnkt-bFgmhRmWmi9l~OG4#Or0g=C z0j(e8x7$OFx5ivie6T!zMYF#NUfTF6FVQJ5eNNWFsKS|kV|d%Uxpz%_9h{!X1P7FC zQLd_{TTd~k6wU@)fvNqCFN330h*M%+GC7Z_N}J(8tFFiSqaQfe!pmv8Y58^En-AZe z=OCL?c(-+_{`FJ&hNgR_$Gu{dZ!w8{%f(_x1?-k04408SS? zlPX8Ik8U(?zLIPM&XW@-r6xL7)y7NF{Vwh_QoOXcq3&_#I56TS84f6o27BETx>GW0 zOJm8b69rAwk_PPYtv{In~~!`-zAHC8cZgE+NouCGiEpKQc0rpU5O|FeLz?Fz6QlEafm z&}Q#r^oZ<~V1*m0WSGk3@~@Yjj|!NrJ;1ma*uxJ)g?dQ1_W@NmSSBwMMtSmw#`Q~n zPWN420`Pxrs-70@SpR|xiyhcNJ~)!`m=Bjprbs-c$%#+8Qm82hrEf_S+|0_ketFA^ zC=b|*xTbey;`hg#Eb_33*#?5IeSQwNBx(>W9Qpi}s}Z<#6u72S&-0k)&j)%6KfVlm zQWqJ-w)q@`ED?>GayI@Xu4Zc2RtbtjjFdYD={{7+5sIJoQO-W8iet`5%v3du9v9s) z-|LJ!lS5YJLMi8alSmgByXFm8{$NYoc2PHH&fL$;FRZ9mSB9ujt_G%BIE%H^O8@KN zsV=5`ZgU(ndgfZ{O@?hC*|QfAz6gY#Yqw4;CM+{9y=dS6W0da1%|k957eA|_17u5Q zB&vqWbIL3YY=>-=W>AbDPC7n_6DxJ1OkTb4w|0^=skV4v^aIWvY} zlc5F&N|-sIO;yB?Y>BL7b$pJNCiU?-ZF(7-UvxE#jh`2IHN}+WE*8fvbP_#Dk6u3r z+=6=h?F#@c?b|O64|n@)JzuD*REr?2@32zx@PTesK$mTy$?W-48 zOaL%&2k}hOdMfl6^^P2C7R z;1l|O=qLMSTZ25PpDg9p%n34N4fW^BVQ>bWol2t}`vrGLpEiUxkq!5S?$-TdIzb)dv;KiznH3xX79Zn zL?%`$-)m{B@nm@qMd&`K*CyqPq02}r$#mDfq3hEhA)LHLReXm}aqd{>n-9KW8({&!PXu^@otJV3@6kWLxF_G>Dh1kqJ3=4|{LKC(0RPZA ze%ZN0Lz$hIuw3R=rD2O{#(-2znq>j>mN=~8JG(miQeI=_D z5fk`!h$uNDD>9n4pMW0M{;Cc1O5JGmO(}DoD9ki})#Ykt&6K%z!S-hunMQc+G!L2} z)k`Gk3jURNqniJcFztRo_8Mn&Aix8Q^c3!*!3C==DBh!dcTc0UTCpu>YrUdJ)MMp% zGh@Uhe4`@+?+JImK)Swp`OBJEB{9^26a+B{^{sGQic??X=H*E?Fyv?N#oo|}`? zd`N@zhJmH}?@zpoMjNvEdP=dPM~(-bl#xU6)jV5n@$$a~uOgP%HYNh9yJ4_&TjxdA zB~+=kO;s{XJUR+U7@ID@C$R|=yo0d7Pon{2{1a;+pp1>A*hI{*vQ0f1L?+uCiUP5SCMQg=M=5+o* zb>F!Y;=RQF6*i$5pqO9jQ#&NLZm4VjEjD;&SUT4xxm76YXCl48!0xM3OR`*ot~Bsg zVU#2jQ>PFYweT2vw{q_15>guw6tWulwIB}_>&wPBFna?fL!sL?FfeaB!~A}|F!3rx zyrLF7DT8lkme1TxJ~(d{J8>4sV$%h#d~rHsjE(l?EnFDX7Gd!E5}D|QM^~o>`qDdPJso|9OHA1b-9Nm!yysNp$(?VD%=3{Qh|gq#F2~ zW0AzhevJ0NvR8JCJgh*UR*-7gBxVjOAVvN7KkknYcyP$!G9vq-6qp*~Z(q~YPv%G@ zV<1FHh}PER6D_bZCW={6XPe!(ABGsR|GD(sMd?9hd&|EJDZC|lujOv>CJ6;4kUBbV zcRXyqZFy|J-fv$5U$_F=yw8TLAekX8u=oZHB|)^IYSEU8PU=JDq26$;&aobViCg%s zcP{ll90isvOKZLnk_wKuQ$k;fH(%7Dr-#Y1eAE3FEGg5~)>NmPK)k&q_DIThI$#H>^pZLSB9J4EC}jP(ZZcP)2124+x^#pX!%xQ0|LumQ)wZjvq}$y9QBc5`N~&rQ~fT~|v`t8Y7s zuFyF?S{+o&xRaFd+mO_0PjithzSIv55YaOcuV7E)9!W^WJg1~w8Ed$?lCM{A;yolY z+fiX|W-u=*&TC98J;wpmy*0h(-KB@3z9HxaBDliv1xY~sV}>fyBMN%^rgtSNSAiag zN1#z}x4Ov1-kc(k!8(!b3SMiZzQbcrResTE0j_V{3CjrE8I<*{j>R*^s%{@Xkn=jV zcimHBsN>t^u3sSHRY7DNxWF*ww>*}IPnYlz<~M^qTJ#q1Zu4Wn;i{(BqqV~DrIFTz zdz_WUnfl%~y@_v_;8M57s$5I`K)|^7Ccx~kOn1WmLt2>#1%8E(bHG7p z&{m|zuN!im)tg15ZjaOXIow6=fP8k})yV5R2vdEvqIHnw}%-`FXVxy(yZbmPY$>u+BZyI_-j_+;KOy?A< zY)UXhc`qBW{C)N2nq7y%Awmw_R*}*So3$dZMfC)Y#?ek1$ec?W-@-F?SbBe@ad2g(zg~EvD(CrB5zMcq zX3>Aspa|CqponIUJMKi%#gd>v(q)p*ngq7|EyJ^N22`I)7Wnm<39K0<#?%QZ>?<(B zf%aV1CT(FV+!l?qeXz-^SVq!QKoi%_Tb`#KfMIxggCutf{inewFc&CPKb++XBYnAD z;R+)lp^}9|$i!%{YBfR5I#iSxPhIaZq^()M@eF?ivM6CU)b3!>i>)ylx)q1lk|itz;H3}VWde_-u#1;b*A ze^L*jsH!cHPf+gGa z9P9_xzK=XCt_ftS*(JAwPww_hCx6-7?ZXJ;7g$Y>crmTe47?C6?2B<%c?q%DTS#Zk znTFqnY+&8Tk(lqfhFBFBR}UM`zNc-#ex!t}Q53H|FVZOqoL7=M+8#Q?L#8@C3|^g7 z)txDN@Y5VD=|#KEdjFPM1!WL>T37)Twz>@E;=G!mrFH}KDX3S69$6E*WJRusXDzcw zN=&>emQw%M0}hGj0B)`G-}Q}}+uj5(XQgdkwwf@z_0k_kC9a(V!j%?;yt$|Vzft;G z;GI>aym^sb7!87QpJRVJR5rbYElaV?yaV>lukiJb)6Nt&Gsi_Kr*37Vbov}@2Ca&? z6a+)zZ|aa=?g(N)Glk$a2yi_9K?p|UDGX7FYRb3kf%K-cQ`ebEP^f8+tAkKe8B7VB zMusbcZW}Y7DWPuSf)uqom2ijv)-cbMKBxN+eCFs6!lLH#SZMD&=exBM#driu^>}y7 z2_^Xpp~Iv`a!WR_=)B+d^*5YHc_}A+L)D!FnXN)Dtok8Qkw(8MUXaJC5se=e1IKuqu>3CzUFYWMrnEM*B*aygX88kNx74CH%-L;nj8bai=h_drQSate?H-_ zFdf4n57|Ny|6fZ1`bYhh*H4H=o+Bw%m=e(b?y0kgE^uhYr{#~`yiYc! zDk6(FeTKwM#?=hMv!z_olBXhCpKcaPjV7aH{%MBRPZOeyW2uF21shUK?_Qo!0wMkR z3zhiHc5$x1P5UW~tBcIl`Cj`xY&uZP;BCd=aOD`*$@_QquaU~urheuCe`E9V z0I^pdrR#K>geHL>kA!QicSYn&m+C0yZ&Hf5)+qoQ5z*5N@$&vkfD!Q~grNJk%=n}6 z>N83Xi6;6V@zK$fw#oEXLZvGSF zakr>9o^vv3S3?BvjbV8=HyV#sUaxF|q`)iR>2BTV3-{`uQyySI=0)u5%Tv~q?&Nmj z$SqILP6@6Nqkn=y;?f(-%Q{a_^{q$e9kG@pETEz8rvUCbG#({p_i`c74;JV}3OTd~=m094c; zckbjGx^ZD0SJ=x8iX`OXltQ~H6<|u)B8GmKMf`}Da#+{&nVLhVowc_6k`T&4Ved+$ zizj>G*i<$W%{*xvIqY5OW87C;uhqqgadydRvM6VbWXi8M+m+zRwlc&rIZ+}5N|*uJ zK5|qj($!`>9d}sH_&SLm7Cy7?6c&0-m!pNso}{zqmZXg6VIKpVGh)2Wn2&DxNLlWNAG8M0&&i5*MGL z$RJzC-hIWFUA;%vq2G}dKr^C-0oF+!WEC|*AK+B#zwR9zKGPg1&}#32L>CZCz{<=( z#Z*9#e8$(qy;9dHc}RHriuz-MiqPCF9}3YWV^!Njwtmw(1@V{8MHT5N(hXoF zya$52GY5Z0Q}IB4)$bFHr$+o!vzahH?9}8>goXjkK4J2FD3=L}?E6e;misWxSLqbZ z{%}6RYad5rq-Ko;}Cl3(JBwJUY;kXeyo;Eq+_1R!hsJ|);wrghlvkhzy&b__PiF}H8OM& z6-n+E3X7}3@M|Lc35Lc*kd0B>Axm6!;u9m2q5}etVj|y+qOeb4hxRd!toKkaJPT=g z;w8Dm3V#s;IH#}_C@|RBMr6{m7nfgx%q%Tr=xC_CbQ`#}<^gGilC@qX-mgGMaqG$fm zr;eZiUUYIU4Z5x2IZT)zVIrXQ7FmMv`Oa!2eHw)TOyYezl^v6p4E{RlQf%9qIJjpR zzr}pcBe`ILKidKC(CFKZM=g+ku>?#iR|oe@{AUli6@kjDTB#Ac(Sk)OUGIH@Zy!_lAMPumzZ^jwY4uw{ zt02n#C3GlA&)>12R2BO*wEp?CWp$-PQlgKpk+HSY@`N8OQNSCsfEFW6?tvXDLaYK9 zsZ*wgM-VQsfm{ND4$x3B|CTSs*Pj-G?J``YT~#U1Z{tQkJS)vSx=or{+RJ^H);P`_ znYnqC>&0vimmvMxQ{W|>c^H)%<-A~1Zar6w&Sxyu4RvxySvV6PwF!xpZD@&VBx0Ay zf&JqIBBj9Mh%NL7cFd=9fi4%`vgQOp97YLM?`1XIEdFO!S&QBe_KO6RMe-xktT{W3 zRxD#p)lPdZwmFdnX>6v{MgWvm@Hx;#Yt2?K%f!-!ory;w3-Q*95_|32>gC3K%7dAX zw~>@@K@Y9+`@H@E`aaBH0)8xhB zksW3aqMeoJXbRD;D4?G^VC_0AmtFB0X%Ji3e&}&Bv^3Lht5w6&E`fCC=;|0m3GYis zk{i94Zh%Ry`ZWakmg;X|DbrLk(Y0Q(0OLlpa4XT_Qgfq2S?oogkSNJpPj{Hzy+ics z7I%EfG#M_4e0G}l`|1*&-1{@JhXh8~#P40yfhjQT*dYCut$AjfFv{4-cm@#}(u@(- zhQ_YCqH&}^hPEAf`ZGU`3SsFYk^J2duG;Le*w%va!DFP*?=v)q0|}E<)66IZ>R_5K z@U0vT>^;#_itKp9pF87s{9+D^Nts?2WU{c(ut62fm=jog^TbT54J{mCSJF&_T?zwd zW74pbpA3Y$M}kjU+g0=lIJr4UP#Fi)pQINp+H1XM(!J~DbonWv8ON&8)|>VA*W!S5 zigK(iZH7YpxGTYNe>SQ8{NT`tA&Rs$4OYTDj-BlDpbMlZs5`H%;YMBjlHWo7 z8Amf}TE>Hes@x#{SN#_2aAjg6S*so++U1On)f{b5R>fJZ;Y27v9 zi!q-rWC5&W-xWwLuLYys4V4>~2C!uC?HP*G;rn%jqm$)?&u z47i=t4_a}s6lnZ5CNmve+^e5*McLXcoN8r?&>L0cD4 zO`#}5X-VuSZM|VY;$oIqta1j;ohJAUP`EvNxP=xubpjR#OkU3J=MB%n6dbeYK_{x=y zd_U$Mo6Or*^AcQ?Z98`}NzcDDtLW9%HZ7uiB#$z^M5Q9&snLgiJcuC^Yd>AIwZ%Pz z^i82-nGyJg(((PdBdEVN76GLHTDb3D->KT#+McJUd(QTmJv(!{pNF&# z)!32JgZcSOBUs+ue8hYz*m)7Gl|mj3?@NVTL#Z^+rdsjQ3>*=y^s;cE^BMwWov=a-us2 zmgBZ?nSRsMO{2JyO(Gv^tLB}HCuG^Gtn;ebvk*-$^RQ7dqC?FH@ayZd+RuGYv>=RD zI0p^XG%XmYPPlI@{SQ_UdAY4fu+;?6kM|1OkL*p_V2udYqt4?jd{cM6W8RF+-wF401LcWp{3X~GZmRFjo3(W~;x?$>*f}L8zOr-LKy(TP^b^f% zbdeNf<6#bI75t$&OD0##(3niiLHnfYHhX`Fv-TFPr_+9z(7}Erpu|${isI zB-A++X+;cAyZmnI`?lNwc14_)I8+#C4dN^XWcroC9rF>u8T%_W)a2K!$#eJ95~S6I zbX>1R77fCaabI%|SkeyESqHRPqP{&ZW;-i9Ii`3ZGpKsGifrxAyEk`(igSbDNSaD)maQ;0d)o2?H3kHGcJzKxO9dRMzBIWEgg|vkTOfhnQ5&Vg6GpO zvMI*zW9ROefgGMm2Z=JJCfSj$4=LWlwKQlaFHe~9)4sw8a?S;?PDg1HT~K~fMB=5$ zSyHvC#AazmdthQ}OyZY}0!*SaSl8keaQGl(?BlD2mIH(ta^Ml;UQ;8fFR+wUlavfV zHctsjDgfF#QXXz3AmbLU1W6ow>#h?Q^-)ZcWh_JRfZ#m(bB%OPWwF|pNiZJ?X^^%B z2>t~9XTj&Qg6lWs=9cFg4r5S&3n#`K*03<6z~Pq2j3eu+jkvE%#uNTkEwLEy&kfrG zVUDHSf`uVg*NT|d5h^DO&qjlks|=-C%RliFieA`x<1Z3O_6sR&s<5#o*8dDAwwc8c znSTE^9M~1b8dm?AVOF}XuBPhFcQ!tSnQ&BMYF!3cF~ZfkJQF zFFFX_yI>Q1S?xqutRySs5I9jiFHJ*Yw#cc+X{Z5-lpQ;?H)@WLoHqpJIk~GlN>pcd z5x`BOqnk0FBs1_0uq{-VZD|ooM8lP9QSxd+6FIh3BA)VQ0{y0RT^Tkny{R{2UbJ)= z)5eRJEMU?h&zw3im4+g((e}JphY0e`FDB2XtID62^1r*7gMXDehD|6BDtG(4b@V zSKWMi?NPqiCiToV7Mpwzan8AO>^|X>g1irUp7tkV8~_%pJN-C`q9zjjJp5sL{C8h1h<(1jqRkh|3c5x%$(Mn31iyOjSV9O?pM5amea|{DUbGO3nR}fZ!gJms0 zkZ*A5bapPR9~gfiM`lZ=&Tbx)ZUOx$RhLBS76XNrravBS=1~l4i$Bo8wPQIy%9Jhz z5Z7YTt89o^fi; z9)p982f{{EV>`p@%DQmrFh0;xj+m1EYHvs_RukqumE-OURF*dq!qdhGCe?TeoKP(F zXrYF&x7K+41`7+tMb#23ANAz)EWk_%bJK?q)oPytQ!pz#Pu1N*ky~q;EissKt=#X^ zXHYa1U$f0H$T|xg%M1&?d+Xscz_~aC_m%9yBr`TtIq9fjnMkHef%GL3I3n+Kk;$!le$q5F z2Ye_h*P?W%qHs;l0a;r%#t7r@9)3`n(*a5BvQ<%iN3!c_=fNiu@B<&TNt*ljqRkxvh9Uz2t0JB zVVK>xES7{NX=OIyJCLbUT#^w`4LuB5Ff02$Q*37%jdO5k!S1ltpRMG4y02day=%Hz3bQ4V%_RCbHV|$$ns}KbdBNBa3GtE*LA??PN}V1rp)30d}w&z-sI4vK7%w ztGFVID{M4_Z$V7I^lk@{b)jqv2?j=bQea=3=?+3;ZHXVX!o6V&ncOdT6@FDX_DM#o zF1Kbo-IaN&AalAa#A2>mFzm}#t1IeM;j1g(2M*TPO{ZZC9$jQAde2?z9n!8K;ZHHd0e)!{wX{B)zkqG|QHJc(4WxHl8MHLQCHB;N+x zy}sVGKsdaMKhRbFfd7kv|Ld_JadK*`IH^4^tY47t>~SKzzh`iq#V0&3 z8195JMuv_@ID<>CHs|mfYX(-|UuXu*r79flc#XX368(W$)s7mg8qW+IzQxHJ(XLFblE)g8Z>qc`@|nq#CWmKx&TGJ8sJd@z za=&g%9i_<`wB8iTv#CgdS_$%-(r_*@@Qmr12XEyKvk^W>rSZXzj74l|=OrdU*`{3T zr)g-{%9VFWYsdhr<^=+Y9MtkBuRXt=kQI#lx_M(d3i=!`;rava%t3~`Rnt1-q~C?K z-d#6(n_Mh`{rw#onv}`TThCuEl&*;4hZ1MPw`}X+r`b`C z%#%5expRdTNfSQtI>ys%ZILq*38KVqgc3WXic`EI=n_;1wkY3uHRIs|e|*jk3*OlG zG*Z^8Xy)`w+Eb&CTbiM&p(d>j8(Q3Dn-x9rgyOn5Zb;ynVX-f;w-=MN9gL_7fw|Zi z(mSHCmwMMQ1J(|{b?e}A>!!=+7KT^Z7UxnKqkdgNs6Vj{+GgcfRZUayO1Qc{QIr3e zGj>YlA(RSa6qzbyYyk`^?PfRDkJi9j%K`P+fFCCp9Omfx*Wo^rfK#XQ$$}TdQ8wdU zhHa8=VXP`_hpVRc!_)+r{gM23N~ocgv7DLrr_WNpB~J=inn)EF;<337^~{L$;OjvO z!p?@2F?4#twtC@WU)X01SU}xQ{^Zn>+|QjVThGiHnC7qa%S7J_NM^ zQM!o)MV(Uid)P!Fd=ocjsI_|Uz zAabq51YTRC({k1`?cTjzrGGEl%KHC_jzyvp2b#6NN)XoJF+_NQLm9P#)V&m2u4 z5Gv^9R}oA^Kbw62XZgB+%F=0-54B-}^NvM9KXloEMubb)LcuBMCfXFH0&m`%pAwM6 zW3@Z2;te5V^bd<|)faz~yO1U5|1=xT5f#&3>elS4v{Iy$$-SQWr6`7)w&C*jr;?q2 zr^nm)fN+}otAj2c55C)u?_Yd=`QM&-KxFm&U{f}&CU#2l)JByU_{t{7<4Je@8a@BA zeiBp>4z%by>Lvh}$WABMer)W6+ooaN+%cBuKV$R0OlK30nhLq}@(n!uh}WCi>xp87 zvP;kfCILn_+CN|F#p_@4x(f|=i1ZYG=McjF&LQlASo0?f>ZYm8ro}NdI&0_!6b=xt zSAo2Ps5egg*ARsv`bBrNVf+}V!V2Gdqerotn(pdzrGsos7+AB!;0a< zr}v*3@SW?bn9?KJtF^TI+zLOJ4=vP-vv^m8`Ww(|gDS)d?bmS71Z%A;+{C);KG~nG ze*h+!L;j35q^e!#tSA{@N$xOV8ojUYb>v$zUw-ZNh54b{Ejf`WV_nYjUMvN4Hcg8y=0yI!*UiZAKI(O3cfl<88Z?=Y-pPKHq|lnNDvg@`^K4 zAfew~)t8Ttjysjf+8fmMO}mvWW=A{*zs&)wAIMRj_Zib&=#M~XMZTs7F%fFz09 zX3KF@bO}5(vF4H#MvHV+6&1SkD$N+6S6^4Jmak227Lg8bHnlje)|5cQCGGYQ8^DTJn?Afve-;+VAaEXko2~s?Ints1c?Vq@$eqPZ%qZxxZs_I?@Ol+m26-DA5L% z7vG->W3hEANVlq-udeKM$eWYx&;ok*c$6{^Y=@_RzTldQ%)L!R!gXDxqwA0j>8h+6 z{3PDX>(8)_=_%RECrGiQ0dMuF*sCE(y)8E`pEii;`pu&)^}?ltMoc=Vt=o{!N{<;A zTcVd$`!Mwmn;n#RVUrWH%Y)%RHhT`d$Ozyh?YdeX@svZjL0gsOb>IcXJJED!#w0VJ&Md{Ds8gU=n+ zI@VIlmo8+5tNSMEbJa)u$@c0C6;0S~Q5MmDOLQI>pId0RnLC(`qk6gqwhPXIsRE7d zoyeNKBA2Wcc(pF7`OZpNQL5|UdPLSXJTHHwNX!VBtxF209@ezlSoKso8b#~vUcd_W z?@@5qH_M}*9{XFBUU-`_SNhd*lqz~T(pA&Fxq-^|tL<+A={&g8(CQjOv=cG6!$x?g zN6lom;KepTsw-%IJ}PrXii3(nz#*>Qp*$`fu9y$g3y?}XM*ag+MU@_>{<;jHHkmA z*0}F9QTFXY5z*&whH-rPvXtIzZLFOtP?R1=@bOJk-3o z8&3AyVcO*tN>QG}oQM+^jQ)Uj!Jg4bE_NMvo)$;ZyB@a($0Pa>c@vj`6caaV8{+DF za~8Wq-jC@qgY#-5Cld4$Ma__1Tqvkd(mIrGO>}m| zX=Pw-_!5PNT?)T(F7vAsMQcoztv=O=OcVC3h=QQGtbhX23vhLJ1u7u49H<6A4iWTr z{Ma!I;^BPzHFDUAr(;uk6e4gwWb++i3F`$Kw9r|1RJ%RFMJO>{5GDKT&|9*%M4ooO zVV)3%v;s{;Q8()nz*qoDx{_)`ATv_U9sK+Zf9%(+D)P4U)!u3Oy%%Q-Bi~~WcW>Sa zEC0;65t810A#^c!ls^v)qq~BSoVX*QK?uU@CSp;0L@78@OQT)Iq+)$DGxD}7&>u+w z<-t5L810H;z-pIqMZzzesddjAGa+u?@$*ce7P_o0jV}}z5~oZT@xFHi_ZRp`j$q|_ zhZGwJN?lgrGf7N2Q)n%^MBH1Eqas?YkEk6jQwh}HPib>ZTB#g>6Xl4(Sc0wTL^np2 z#zpu)(9DrW*+uW=l8jOJvGR#B+7~uOr{hX5kp|$l$TYJ}5|13&adLx;iF<+Q=uR=) z(|Y~E*MhCaJCn2&FX-+O#4mYfn$p9gx!-mvZE)3QQJAVrD=QT@#q~+t-a>TA_v938 zy_&-jk|1=Sfb<=vjG2S*B?%dXK@Y7_elT*4RO^w+zACawm=z5E_E%Y;4{uFSp?|9j zBx)p;W)3dip#%SSX|>4}0$@y%o!Xoz+R!aqd=rdiU<$%$a2_*sM8Osm9AH0RwL3q6 z^9Tvkk|{8eO^$)npT_7Yn%Kg38}>qi?+cCCsFRcL2cOpUUtd_8Qy(9G?9xIgc1t7e zurc=Ozc7@_Cq6LO4)3O5zBWr4W*U1h?w_m}IBvsX!+~Y4ZIPFRJOpfv3C^(5rU`D^ zsr#m)rnIi%(S$IEv=eg)Ju!ptddH_RQB_7AosKPh9&s!@M%8s}E35pv=2vF<*0i!8SN2SZPFvg9wIFYOJOVTaQu8?d#g&JS~)4}h0VxZ>~2-{TkRPjySIU9&4qT%<3fKy2nfe7QKIPk~w6H<6_d zyI-kg4Pi$*Y}qc_JIHQJFeFWH>qbb3oKM_ewu2ki-U0<|Aqn*nGsPAf$Z*-r`#QE+ zz`XGm7~FDSdnarLxMX_EqvCGgfKr_*%kJO-$JJ*Tl9Y?Bo`;=H%r!|d?}iouI|3)> z%*Fsn@QhcPj}8f#z9_p)Lz6rZQzsvWVh)YU6*WcXS$@4WyOSCL0e z7==~Y_@=Yg8~6NsSKP|O#`yd6b)@i*j~tt~owr@T1iEfFZqH0#3EiOuaEX_PYlXMW z#c9dY^;SxD=mN#zV`b4Can%edcN+aWl>1Mpi3SO&lD1AeAj?+u*+*|{=Pwe{1fA0cP_g9 z`0$w*ax@}L#u&LfLb)jyaKe<^Dp}@6Q*w7qB)iLpRK6re7rRja>*g@ZnLx*(H!s(g z1o(NUg;qqSmo~Z3h;h)4HY!iGRxsq93z$s^be&D201FDgxRB17g3) zrXrH(E^h)!){jkCMogvMo8oG8n*CpLe4^ZnOmO^VNwvZ=tr32-gVp=hUCQGY`II0g z)TiwMrjSMR?6r@Fr;ksAO~bSvr0VFjT7|K-wXSH)nbY7rhTRW7elG)d+C~Y5_*Lwt zqLSvz3@Uk3Z)g2XwCVY3LQFf%P>fe)c5|VL1%7tG_A@YB&Y=3mcoWMG z_wdu^!c-F(ciw;>(6K2piquWDGID(1<(sYLCe~=)mO17Osn8X=6qL$LaT;9 zN8^BgNsVq)QnqKS5Os$D?IO;_?6;x~-rT@3c%803(bN;jR<0*0?nTR(*LwL&4oam_ zzliP3YK?P~oVADz*O+2^LF=aWDQw==x!lAA)smRJ*c^aqu z!Ua#Vkj&nsA!COF**Q4&W8j?mGTV;TJ8)vo_-oW~M7rgv(C|A3^UC}Z*S^!4Su2c8 zU{oU345zeWqk^lGSbhUXM}YaS_M%}A4`f73rugR6$}IB}ecYr=U*8yYA8euPuE|{7 z;X4?TQkyrEn}Le6^Ck28>F}0Lwg{7futOthTci8O$vrK~V@38b3sCBz)#|n`+J2^M z-DbX+nux=J4mbOb^_6` zL{0O@cxu!g8hjCxMlXyXk$m*KLrj;LZ#CP~6XZfzT4mqL55BUC)-!*M9@vx6We}Gs zSRwGATr=8Zx7{-0*wq>@%R_ltP58Y4>y?m%(;b(8UpUX$9kaiT7Ikp~j+WxWI=6=J(oNkg~rTU93R;d{+gG~cMMD-m>cBcBAHLj88jNMTSP=3{)l`^!&^p?Br^Du)wvs{coITee#nwY$b)0x(pO&-nLDMYFlRb4Tr|$k0%)4GNK=~l`Bpi-VyYjx)a+GT-7+5l(6>PaK*BT9?dpuuElqxBz3co5ZwWa**t@)~|ySb$cYt=c{XrJc>Y_{o4TokHt z3t#N4abpDdcFqcR^Hg!oW-FLN|9D5-k0$q}M1xk|3T8ehh|L#4sxM#E38zbq4u7)X zCE{fZ^sWBGN{|v>Nv*9zZ#c9;s1RQ_d^k=m^-HKBmTrT^ zN<*GfdnBOI7Dj8eA11S94MX2}PIIL}I%?5uPMo`%FDeDUs@|#^$X(4GwZQ4Q$Ugs7 zSEYS0R+m%8P6NkgDvQh`*z<)&-C)ELvTId0*WM!nQ@<`#6Na6ToL#@OZ7+;%$;*`> z72A*q8zIJdiGV#BPA1qo zv?{>`rSL^*llqbgz;FzbCA4Hx7iDRr*~r_4gw(+9HvB3PM0zak)Pd0T#rJg=j~=t> zVJcE7Q`x|5(+HQJ8u52CveQ*N_7s=*tU@75eXb3eJ{k0FdPl$$cG*tBn)(7QX@Uxu zm71akNG~<%E{)e-FUmwKtf~s@a!%i#iH#K$D}TF;?zAWx0OFO}dJN=e>~@Qxv5z=S z%h#oP3MX#J{JN=erl=2<{x;%}`@J!Qt7&MzdGk_F6r=6K$BAX5-tUM!ajo2`d)%dt zg_xBu_|$uUnjRD6m?Ru^uqZbXOLM;P=VGs;bW1nc#ns<>l$XOU8OO2X{7)gqLHfVHqbDPC^>zOe^QC5X$WVV=mqMd|S={ z{|G4GC|bub=rtWUK|lO%R<{5Lw}A6P>|7L{e2B)zubDu<#x5BZ-4lV0D>`}ISXQCP zdbF=u$U`BmCVJ8v!H(3C2jb$$@jYB-rx?A}Lw5Zlr~LP!SH<|4#-Ea>^!`$WtghWeQS7eW*Y$XGnuoC}046go|# z0#fy^JRae|E7p(zET})2JwW_(s7`- z`>b3=0X!mt+E>}1KGUK84TC^~I1`gXf`D*vwqUhzHa4?0XEn98V*Sfu5W^ZJz!eGf zYK;%6!;4TEXD^Jc~HUrjo$LX2O+K_0N!JQoa%@`oa@A3EtPdjfHd}# zU~LT>EGPpDN^qV84`WFGC_jS5r$iv~4K|d2hu0rLZVEEmzyT}~gIg#;85?B34Sty$ zk-q@E)=vYz>;B^e{CyM!`j9`t_aiWv{&Mc0b!URMgvTG?zs+ul5DSNa)rfbOaP$xX)J;1)Mj`AA?_JkcQ_gr31&8K2;rAR>_A z7R+xdw%5n)rC90UZ>z=3c94D>(G!(a78I|<2LCMO4Ba-f)XuR5`1mGKh#`@?$B6~34sO=d8 z$NC-ufW{nTv-c7#$lId?$XkLQCk6jeg4?GAkXVDJrv?5NfvvKjuZ6dc z|A(XhUk;J~Qi%L%>$k{wB7gwLA3iw#{+QwZK8gaNTtEqE*kB?7B2eD^?^n|^vVp(x z;P;|H1h>Cbu#o>`Ect6_Aix48z|b4qNK8z{{;cVr3FE2$HpqAZ8zAQM2M|;aPxC)l z%d<+l&rDAMpOjuKlKwWhXSF(?!8?P&a7qyCr$2(uWswrVod(J$eGi9osfK==r_R%zEDc%(rvasc*Gc z?R~0to!>*|PeVeh$U{QGrGg`1CgY)l!zDAhfd9`8843&x%*oZ784~ROAF+=4e?O)| zffHiGfr0(}`CpJISUTlWc_%m+7%n6j7;~y>7FO!)14gQh8y4_M(kD&CSQcP6z!e3p znJmJ@pym<0rf@}?J&7xQ2>XeF&qdO-2l+-b+=LGM)6kvOjo58=rSo-m&UhCH<{i@u z_Sn%9)eU~L-(%os|jG*BRs7_>%g@u*@=r3w4AeGHZMm#jh>z@ zTd@>S^mQzR4+h?)vrZCOH`8DYjk!2}sP0a3s4q~{OOsocVEf#_jqPdJE_p2SS+*Cr z(}++-D*sBJ9B#!ZfAw$dDyAbYpfNUjjda=XhjIY~FyHjpT6o!4U#`wBTFBAw zA{U#$ygE0!>=A7S4}k{{Z=EOJJj7Ce&LmqyJOlsX4`*KR?)9orB0J}K@qo`WEvT_y zs6W6iv`IE^iJITg+ZI@#sY*I2k*NEgZ;O@qzBa|BR}b5c7!2{%MQ*P3Bl?1?8jTD% z{ZFF*H>&_}h8kjMFffW#QV%ZRraHPd&Obu8^+tv$R=VhrCNWtsbmlg#My+_7d1yv( zht>_I3DR0e&vUDidE48%UV+!(_HWo1$~CsOrA*IqVCvx$f48ixSx^7;ddu$ft0DJ4 zp|3C2pprLPDKK<4qakQmZ1Jo#7D{ux+sQl8bQW?`Eh&Ud3xiQVwv!|v=2%Z03YLYz zfTl|iv6zxdeYmsXAPkxs8(9abIRQ7N|5~(>KZPi~sp{HfYzYYxld0@zAUH`xahT-~ z%Sn#TJl0m%>`dH{3U_rcCH6P833`O@IL$qTP0j72mh?rk_R}`WP#V*fNxMN=FjBzYuKKnuJb#LRDEO2E{K274sL|unwdhV!Yb#~XWl%sg|m~nom zIfkg;)(Syww!FmBnQD}!j;|xuDN$*NHk9yVmr1q*>l62eK_40zp%WT3lZlY;wMdjm z;l)ku=unp5BU7`kK2#yw7wek-Td<_A!wk*cm8&V|ls>^R+Y+|Dl25)%SEVf`+xP^8 zZwxG^^r)LqiDPtu1AIJNLqCl?b}7y?`o$Yq)y3qUhR8%fRq?;TURV-gJTyC#y;dL-^rtV$8v6&{i@0 zLzFSMh5{y;+Po$&%QXn?CLw9jiuc(4mFnp9Lew_ZG@*gUd-Bl6`*s(lzKiS;nkU75 zaD4+Nga&nZ9I+Y*;79tI}fm7&n5cDEBunzWobp&H4pb!zUaIJJr9!F%BPZSNM3ww2zn zHmLF?xOlW$0(WP5W3j=|q;mj$!-vrI2ef9=y$o8nCu{Y+pX(`2%U61kykxG-^{U*D_kz0ms%)^7hn2LE#6`1m=1Cx@w{qtSGC zCO zq@2X(Q3Cel9vv$FDh%t@lusYym+fIKmT0_&{YyEV ze-IIt6c#1nsnnT;yHj2u@pN6aa$sHje&*wk-n8ZHef#63id^%l&OC_d>+%eLefWFp z$AKn67Va0mVM*fGZxOs_!hmt2t};wO6e^AiPo72<&5CW3NCoT*D+HZbLh2StmkYxW zYgz%<(;sjt#N}Yf=s4g4P%ltpKY1oBw)%LK&mk=iZoSmQ|NTE)dHaBy>hC-8e^4Yz z1#1QWf0Vig^B-aUN0CISzJ4^oZFPK2jDOqqT<}vr)MRMVKtUi*D7k8_*3vqMU>h5n zbQP33zRV*o`|hg)E66YWzkBM&e~XosjQ!8!ey(J{C|dIc_MXMfp1tjjy|Eqw_q=Z? z{oUbs2va9Iz2-VkKLbzAorpdk?*+l;_88G!dQ=jzekGZy#Kg%SSP&`!vw7mN@ss*- zt9b1W2qN%U;!q)VA(^wI4vxpOaVry}38cz+oJ^+sp&<_uP{-&q!{_FcUq4;Dt{zj! z7v+VFbx;EN8scyxx>B4avSC8@&pcrwdny0id!HBIrU$529bLsQg)SigPgA>5ycx#xX(R5Ew8`ofeKs+%s~=ki3yxZl06C9F$^7bD z?Z(=aw5U;UbiiQSN zpU`UpAsckELKZg z4Z7ZhnT$QbQ$Kbr#y=D5)faQU4_mY)ldG3La1Lh*DJ>&6y!`dg%JeRohMG1=XgTnh zf0OLg!?n|}3F;$QuK;*)vLnwhA$C2dIJ{%Gs$%Zxm)|Xbjx~oH;I1)F*l|2mk)FeV zO*seG+1HFT(el*}0M4N+O{-6!;j9W1wY5xcdzAxW_cDT^JN~Gse^HUH5`rOc#?{6K zaiX(`Q{m-{N92J-a3;m5gfm|cT>hDfJnce1B4aTFwXdXyPZlIb{m%Oy80_mGE#)sM zV~kF8R`Cg3h!?-;il;Xy_eA5B@XMLO+xdu^$Huv;{{7DMQYsYof;+9DZRZWmH<8

Ca%uA05UTpm-fb`()QYj+OI z3P=Cz$3wzP{iK;C!SDITgeR7zMCo3A;J|@+Q9#1CRNqi||4j%jBmz~Zho@Yaa5Sf} zY+T9u`v519{G-7Az{}zeM(Qq8?wBXpj^Y6zr$HSnOI@H9KjB!KsVU3amMRONtKzeP zJVh|jK}^HmMrrGzqM<)0OiXXR!O!FF;mN7*ZHW=AvH3PBc}B&&z9rNP=!w7@bAw}) zm@Yeg^j7mC%k&RMTnI2Qh5rf-$Nvs$*L$`3)hKSmX|X=JS{_cqXo!|fZ?z{6iNV}p6iA+ z?)T4Pk>3_F-r+{R_^d){)q2;VLG*iJs3|oV#pa8^D(PfW!e7h z6@9ALcn&roA&-!3Q281cb`eI~AV#9w?3ioZi)ZN3<1*9-`cX!7!UrMDmS0uo=pZZH zW~XW_yvdM<$}V>FqYqiAUurd3{tej0nj8(~h`BpQ`$whJfxt4662HQs_?k@nur`Vw zaq!QOO8jPdEqi!+yd1ZgmGww5AXuz8y;LJRx6=#AU|@VCU|^*GKl}ih%j~k%>3XBD z33{M0Rk7k{Aah;e?xR{(pJDLMZR`nu@jGPr_q;sOUvkzr^MPl>^zUJLpd7G#{u3GkPG!Qc4qv(4svRG$Cb3G*+1c;H^y$kVHlp2WYsvdnYhktbxyM`gON zmQ15kpz4|N$m6cW<885zBFDe|vdl9N<}V=iSEkK(H-zs$>9T*%i-~7ra9gIOXO6G;_}aLsV~p4RNXx@K(UdW`sT*K}_!cv()kKZEv0 zBzOfjSTR<~HdwLcN*wYSM%9hR$CgGH2YB{jnKk~onF;rB=}y`{Ws~5dy^uu5nnpmp zqZPV_`^7)rb(5DEM$9tLbWSRrqDAs>+`gUBxtflsYih9@PHOK=>D)CvlE-y} zqYh#pp6OndTjj3Zll+=}`p1TJza<%Zr1!VsyjTDO5`&F=hrV={rCC}xtWwYJTRZ0Y zvcrmk8&28g8Od@UV`?C9DJo(a>p9@H(&nMmV1lIkCRCGE<*SL)?*0z?kRfD|i81q9 zm-0o$^J+&vjveB<76a?*ti}p}N)43X82RkIq0*UJ)ogBayHg*GUbL6YM1|B9brD_F z1uS&h715?~FLo2Q=$05ZF|;;za?(B};rNAv@Z$cAh?su{>*)m}ZkV->J_tN`Zd3RU zuHW{PV`K3-I9bbD8+&kE!m}f6d9vBE5i~!aC^c>KoRZb4Hrc9C#!)bOia?~tWJgzM zo8bMnVWK?Mx(T(Io7Ew+x;49~EEo&v+{u++XPG{_;V5BgF6Yl+@m)eEkKn9K97fy+ zvKNmVm$D>+yIV%Vew@0@AbKTk!)WQAypIOfRit?UmqJJ)(>v^J65ftVbhKf(tGwk|B0v z0^^QQRpF_m6h-!;^4UO+H*yjdjCD24$t7j;wVu(|E0zwd?>~^xd{A z1gUs2Bbb21qRTc{#6!N0CZFEOW=6iejJ-xGitlJ^BffDmk1moz?1jAC9o$A^a#T!7 z2^+sNnCHN>M*az_x-LyUi+cF*wicA+A!0Uw-Ym)tTH+|FQgVFZ3__4Lr-H*bY^Q#~ zrFFjdA()vgvr$9f73{YZ^G2#6$6YvEwKAGm zTiSjCNqXt*ed#Z{8m%}2@u}WO7X%o39-_{A4LMF*=!wFeP6q^n#^aJ9c_0fLOU4FXX2mey`9GYG%?OFfbpGXrhoKD0nQ& z_*x>${p+;kA&@*Qp($drH)(JtswM>rks=ttaFY?K#gGchd1CIM0yAB@AKE__eZq^d zG)h&h^_pM^`(G^L=g;sA|GYs6m){?LGI{IRcR`aKG7_RCa0Z6##t;7j*Cj}3xl=fN zy-QUZ#I83?HCD}oUKeG@ca&BpU8eI_J*;>GygAu8J3V^F7wlWdzPZX{XY?DF2NmW6 z9eDXsop7Tmiy8gg_1ott5ebCuY6S+cDM2l15APvVc+ur}?AIwc?7uDtPrL!l+;EAj8u z@{+^2+cghGor>kWA~CZGwu5T-#0WaH#7OqgpgZ;J1b6ZB8AuPEJJCdf0+_x^1X z>KPcqwcHb&HP)krgBvll%}u2G8F9t_h>*|DwWhO2C*}SlSY&p}r z8mj*h2ea6*J;!4k5WQ)5?v(tDAUozH%q0dc#2LEPpSlw}nTF z)zs$p8u$jAY1K}-i#B z!N4FI5eTcLqj`ZB*USBl&>?gi7V;g)ZfWiJ1?F5)5$mtAz@Nr7%t~7A-%ysQIR4@Oph$UZAb0l;1{a=FD_+LU?9I7&*9l8APv}rj&#Xm9oyjGQRYb_gtw)&%WDi5Q=8>u z_ZXM68s<3HTs?f^l)ZKB4pUdi!b({g@|i}4|HXltz5vjZvOcp9^hY{`fgg1md2@h1 zjARMyu(?8KHa`0p{>bzfx)8XB3$MWL!9HVcO*H3_I6fL}DK1I|=H^gGH^{Kwro>s= z=*yPzpTj=kHOD>yu(OUBPNvHdCi{mc~#MV$O+#zajP1o2E?S;Webx6ur}R z;(y-MXzZ{^<~y4&QP673LW_^YeQ(O57W5ZAGmUR6th9z>EokvL;n$U2FP6{fPWa@m z9y{n!x9d~+ot7yAi>YhfLSwLH$GU_7j0mwnz4m{B1+kp96m>0n_M@*W6;B>-O<$iAlDoQBb@TW z0Sm?-#l;k(-_+vrn_GzSfv*(r`wx~5F}v6LH`;9Oxx2k40S*z4UitK2Hwe%hli15k z*ehI`aW`uhb!%F>O?8vTH`?s2bUkgwSLPN5B`dgkkED8^yn2`VnMKd)8pJOSS)~#9 z9_g|4OlO-wy36XLmg;(2R8A!gfq$rrEXTcfpOku3fKgC&1as2HPd&q958D@&7Gn>_ z`f-&h_@z11$z!{j`MmFk+|s{O?5$1jRL`8&HcRlWzZFs8Ds0@mQ$TMZE1TT4Q_o$b zC${AiZK|p-DSbr%X$;CiL#GRLena`~yxlmDz&bKe`1k14F?%iyaoMxY>%gI?#SXZ$ z2f4CeQey9Nn8Aj`*LaF})DakADawW@#$T3g3j5Ebu!$T`DP`k)K;>(h?# zljt4+=U|elbI;JdH!2HR+ykPRNQYA9zfDCXp`Y@~3&iK(<7xl?^wf;*q^NzP(KKb= zEiR}q7K!LpsyRP=iY-R!WXw!F<9&}Z;v$~%=70J^>okGnx0~{T|6=O}W7c#-4@eOT zzrx^r*wo-fwA6&FQ?aQKE6_S)hwH7Jmxt>G*3Pp+8<}w@S>;@u9-B?HbvfdV(o?<% z`7EuOFHrER*UlTNh70k>L)0^l?0p79&v0=0&=W$hRK_$&A=L{&3Bx}D@{9BKBM`P{ zY&`d0u|97GoVWuCQGW)l~L~m5GKi6;pV9z3}6Oo8ZQ%p zOQ#e-cC}Mpm=J;Cu8U$%UiK zSxpt>i*oQ&Q72F!7$Un#7W4`H{4>th2M>c!NCt@@M9^n9cpZD>NWDPR^QBq}5l@;giyNxr<&+9gIrUa0;)g%!pG+GT zaq(WoK2uhqt5Ke-hi!a_rkU;OEM)658U4=PYvwTvo-5lY!kPI16P^c5K2KbF<*4*m z?1rcAP7JSH{RnyNvuN*h(z7p`w?mreNYs+UghaeOT(Qn7nuao(8Z3-INNJ-$(yIV4 zO3s83A1F#UB1y7O4`P8v41xwQ2!lN%MPrFM#7VWkXaTsW1d)j?t&cpuj`P~{_+V@P zn1adV*pGdEG+!|_$YxLUPc@CI-W|!45l4+~07L$AKqc$-}iZ4AOb-HMvg5u(idDbV@9h=>$SCIt83F|Hm z!_`Z?tle06TOZB_SNm=JX{@~=#Lf7t>gL?3(KAI5-Cr!<{IkFV(~aJlC+gpw zu_%@g5bl4wN~so4n5i0BtN%@Xe?Q^2NMv9C`0q&ajN2k#SPk)iw70X(xwtD}!N3r) zz`*GKX9BDQjtBI{7{mOA$!o1xAuB?^wh=NHp;ir}{Sj+L3M=bOA_76?9CX|&1LM_h z=9HMiBdA0rqql{(#zBPdhxU)jy5(S++DGh4t+lLTGW}x?ewjE+KWr% z^SMtCrWFkhmtSPYo?j(4(sFcX&%0H($4f^?(Frl9c?T4Lo_jrn4(w4uo2~IiHM$y; z|Mad3@zYqA8`U#Nie*;6ckK}ypx2yspJ-6Kx80&Qy-gjmJG#@ud_DAlGrAqa|MV__ z3+ATvZixx^j*Wr#t_f+c(qRC<8H0o)dOEzO8}yEh@j5VkxZHO2P9)g59Fr7gs;V6R zIQsme69NVySC5hiPNPN3jg(gldAkK^}P`Q`& zuZAlawZ;%2;}xbSmHj3VVvc04d?V{CJ(zln32CTaOK5xxoE1>aWx6;Uj}NiXE#5W_8KQvAO@>UyntSIGU{dg z^pPr)j8sOrI~t4k9!b z<1X&Mx-e5885XRON7W$Y$;vtBIZV#KTuRR4%8(IpwbUco1@~q>FPbRgA1L4GS=jw6 zmw^`2U5$Q+|sj#oJl2b6iN7xj*8Krh&NdNmb zeClL1SaT5B*>a-83a&H9Z(rY`RI=GpDWl6c>ge|rof=GXp>?|zKY;0tfl$R+Bd>4{ zPe~V(mkQbwua?v#8{L!bj5iWSa!1b>m4MiG#Oc=@=&dbejz|bHmZ!dwqz4SH#)=t> zfvZvF4ym89J#WYuuCndz1@UQQDg?6| zwPNpV<>B&cpDtEq%Z!<*vO|s=?d`F4A+9IbD1Nh)4qx`OL=RyPXbzd{cz@KZxPXUm zT`4ZrDDOwO&Rd|lT*Ety~i zNaQH4M+G*=i!Z#FV(B6D$vVEGfKl2h5Vnf{bjDLT>sjx*-x!vKyH7mKTXH>v; z_$TQW?r**&gxuP`5%nqOt3w=DECN}FuynqF1pJ7?{m7i3KZUjK&=C_4DyEC?#h0rp ze&9+V3J=X;eu@n--_s+q3P=v2C;@naE9l0I}FH(+qh)(xCv1e6|(-(_= z(?9Lu{BztUvX?E3gK8N~@&BDUL=1HynAP5_Rc5bL`a8?HUKBW70M+^_Pz;p4nZ-Oj zRvP@)9Wdtj!~`zgdEssW_i`eD2Y6FMdDHNA$s(Eq8NmHr*IjeC?~;9&PwgTS(qRnA zG{7MBB&KQ+tmIp^GB=Ot)3UeHl_K@E&fqvBnz4qhSi0U@g1V2Q2CSlqTTQ+~j%Vd= zVKQt43r6~>W7IbsA$Oh1Dv+9V^?lxDN+GP+UUuv$4Yk_%SGBQfQsnlS5{{}qy1HZ= zhIUC5Ag2-&Tyhy;#2|B0l=G-RIsJK<2KZ97_}ROg{svA{LO0lbvVJ?{gpmGr+zD@Mc5FJ*;)%!VbTDN_S%PWScwQb;d7=u=lS zKF5B^Ya%+TV%=_9h!rxup7p4rOg%h`4T-TX*fPRyO9=U+JZhle3AVMm%7~BGj=`b7*G^BwqoB_t;O;S(8|abHTgq{-eb?UOf}_sF%kaUY z#WRfZ;QG0j{Ri0Cyerx zZY*VKtXo)JF^4_x%a)ZR`pnZQl~N)55UjfFYF*eVpu_aK3LOxStKd|RGZ1G*S(&8A z8k2324NM=nLI@G9p9Z|Zd`(HSXe*s?YBS*vd|GiN@B#KN{xg#LADX(#V*$~QtxI-`fv zfoR=Cq%6*UGb;7VI*!Q%zP@n9WjpDTh+r^J33w&-F2JYYr9~D>{+S~Nn;GSfW103y z;!Z&Rhlm1IQrIvs&EQ7@JHd}Up&(J3G!K}Dgh<@6$K?YkCA4dZ=UDOqX9_}I2&&)j zFXK|Ya<#OJY<#X5`+cJ#Lo9gGjcG#DDH$ipql9XTBS8+8;xHNW{(xl#Y&fj+phu5o zR-jxub6;(L+j?uY*=I)QKPD{7KM6fY)vliqVr#rn84SA&()$P*k4}<>w2f?``jJi5 zIQ9Io$>w^=Uu?B{n`ZnxdVc>+fq&Idy2pr^XkqT z(xCWYUCj3_~2>;(01&(;}P%qc%gc=UpZ6357tPtx8j)?-2zN?!I*1_hN4Q-E1HDa34x9ahF>Oh^7R$} zkpD;irTRVNrXrWj{~scc*B;|Q>p#a|1R59^+5d>V#uu1WgqNSdJRKi@3@^-o-_2tT zso;w$V%RphdiHM2`*dMLVq&ZgXe6XzP^EcSNV%jf?3Y&z!M-XEb<(|;>AnRzl4Wc5 zVhf`mB6ht_3tDy3f|tZE>XHFYR^02`4798TbKkRGUppUtuOEuDU#CTp;I)IoC#H}L zjwxF`N%*&>(-S0I*d7Hb zZll?myhmqjqk)oc%O11_vlQ(qZ^Rqe{WgWaT_{F#%{wHA@!seSJ)H!v2aF1DTKRo- z1gbr~1gt%wxeF*ahsR!%-q<^g=1FxjdxBd%+Jhfr-oT$g;-CAWr2aBe=t9%@#HIjh z3V)4JzAfe@SYfLDLo_A1w=j_r)D<059n}?_`Bd)n2&%@y!$x0;_%p4rQ+(*jY^w5# zUtHT*I%&D-FbCl&i_suv3#V@McE`1kagXNU7i*oK|kWsT-%zpBT``V7k1s|8Sw&v1G zb0U;)mgaV~Nbc3=tp9}1Q?bq9zoaC#EnjK#gZ`}3itFiX6z6WMcQ;dMd94g3h@3=1 znlf5h^~pDUdZdL;^IW31kI0b&aP-<{q}JM$i+{)h(Il&i%RxLHF*nI=NPq=X*aCM; z*&W5fiPxT^x5X$W* zk#sx|$*Wp6Q*y9^;#bNYm@dDiuiPLYc*M22$9SQ4`?)Cviud%rH3r&QY4`cz z`eJWkp>k5HuW8)P&N zc~?irEtK>C=FzzR^p+5*y}$Ab3kJaXma^DMPg6dXg=`}mFAh3`jJ(GbJJtNm3?l>D zA}(9I>cf-z{)m)xkVX72)dGJ`1B%lzPV3CcikAnGVxP>M`Ir{xzg#BkRO!j1IcSz% zqiR?u!#J)CPO^I&Ick=_I|4H=t3U+v25wc3(9dbsr((@+mnM^JiwPikj|kv6t1U-$ z*;%okthT2AthUNN>xoKsi*2s1!;rl#JnuUP?~dlQK3_(ePpo-%ivK zH*{01(#iZg2WPVNv+Zh+&uk7nX;Wnwl9*bG^^biY7ur&lpiovO8jNX6%4OwUZM0p@ zUXW6|i{d_H5$Me!jej-A+5>2Fj^m|s-vPf0;cU&EwcbzISIH6+W;#-#s6SQikkbvg z=8d%Pk;%*T)S=U~EQ$4Xe zhbd8<9I>pC>6vDmyX)Tl3!J+VJl>7fq|p7HNt3YGH{~DEq~GF9*dL%T$nuqPEhX_y zp4NvK8eeB%^xp^Wbxbj&M+yU{60DYxVd=?*WFL7CeT6*!KBdM|rV{+P5EMP#8!eQ| zZ_3T3(DCbY#iN%e23J2{?gj4R#6bQ#{{zR01+YmHRiL5a7mkGC5YFTU-q>CZ{Kwc{ z13b(qN+mqZ7)mXE2b;y+g?a zBiF~hxA4@ml2WI%G$Ji5)LmgNr?%MH8KNzK`kOyHonX9>Db#mWT6n{L+%e6^7GHS} zlp&IQ_A~m4!2-CSPTyO_%2R=p5 z80;>2*u!DiEkEowHSBH$>~LsPz z(X>Q*D^c)(z;}dvje54kSRfeu9^)kzco5VPq;IG|9HQ_y)0F!KB}7?G={dzyc%pvH z2oG9zG4Ltb0bI%h>x1*aW#pO!sPNsThs9Uo^#)2(kVM3;>By z{QBZZt_-YfT|{_?c#-`ndlvWu{$y9ilUHnT51JtZB^|}@M_nW(#}_=zcl=LPQF1O} zSU!!W^hKtdGD0#(ybxEO(B}#{tA+zq3uP9{hz-(AqcPKV;)wklN|WMLsq|J5_>M7m z-HXUK@&C4^{%7f64IzcX|39-6?hh~sFvkCtBL*+%!1T2Lu{=fW0@@9`w#66&Ns9gD z_38BCa!^KaWaFqLnOPL?9Cad^8^6XKYre^yI_4#c7l!4TKUGKBTjcfS5IGd*rq}ac zd2iRVw}pYgPY9Jka!8jWOC2Q@n(;f4a(ccPR|C!@5!o@kP}S$RK6)cA+Yr-M|KVM? zDF9H1nLt~h93E!NWix*bt8cERl?=Vs*{9d`x>caUs7%whBWA!~^HqP#qa7(cpmwqA z4rFliF=)LuZKQEJiP&HZO;w9bjzk;P$*n+qLlovvbXbRc>M zv)U|=Nw4^R<-|Cy@%HC9%HsatwXy&0mzSnQExNa#(aAoD8u>hx^?lgNf=A$64$Hgp>sdit5R4t90j+t}R!8-amkivLdg zvL`5fe!hLYg@5dHzizqBJqVxX`<+Jq>%7H5W!(P_17JRtS0|YP;w~yAmpc>WR4rmm zoVJpryUHA%6CD%?;FOZgB{gL)PQyLe3#sQgC!OAHkr3{+lxA1BxFnU|`4<@%y2u8} zhq_~D0t|>s&kayZyF1l@7?3(knZT> zYb?Dzl6I!-N*{j#!5Y#}J4FLw{y=qTNnJ?%^$^n`E+sP}-g`Xi6g`X}sXL0tDf5N^A_D)l04mL`u&b@ z)2%F%%n6X4$;P++CwM?Sv~=s>N)wN2sFl)(J0u>_n%x~v0==_-Wj+m~xA1Clxurh8 zQ36E{r#SiqWHRQB28q*PiP0>D2}-$#5e*l62DEKHQR5pd7V!^cH>0F3yQV>c7wnFD z9;)Qo9H|PxB9G@wI}o>Hx3i)6Bd89sZL^)z-m%MBeg=f70R5jYI}=Y+m3&8c#a=(z z`_#4^)&q7QyWIXV<*Rd6YlLRhuHFzWciur2eD984YHF-uSLlR(rVV#C6{pmJl{($3t==XA zLJSO=gOq!uspJv=_IX=3+1-}HpL9ZF@xLaymUy5}rj*uftxnsgb%!Z-F@2iGMl7hZSmq zAyRbFUS1cqyTKNH9@eHl);&iSzej)&B6RSo8u9)nEk0d=oN8fp1LhVVG{Av$MF^W! z9-myTB}%!~`3;AFQVG9t616y`(r`nNE)k<~5V$=o`ky?kL#PC7 z4|0;t8Vu|1$0Ao0A{)@?X=e?M*tD8b=@Q0Q%@UUOph{+xa8;qxT_z+ZZP_(N;<){~ zXSg=)waJ4kLuD)RJjc}QFN8$lWvGGIM6%Tj>q24iqRbw7Jl zDTMFN9VN-;SWmF%NdcW)n9+Q{8lx9+&0#Hu2~d%8TU5LGE`gjSqzROD8r;CRe0jD3 za5Y0REcSyaVL}O5dO{C8WB6356n=6Sd-M4fb+s;dTgnv02rwsuZd^{vZ9S$J@S+Tb zs%vPq1N!+@Xb%STej)RA{ro$y8@w+(CZ&;Aslxm0dY6n#@B-ZSy1rWUv|!dYINe$! zDT=xQ$aMnJffUnb-U?&3l15<6&*m7{td%i!R`%QtK{*DeZrYDn5O*qF>}dUsNQ!XL zJvI0WNW#llsoYFY=OAZb%hIjzM~e?emVhd&>AXsLUc1c#S-H}mI;IxpZy;4o&yM6G zLx$BvBgT{j+$$nCdH5yWIv?$jO(%<<+uu0X%Swa>OzKmQMf=qKHZ`EPWA~a_(Vl6;L$e|si9d;OE+LcW*o8N9-tMAod@d5BDB}S-HDD?fbLTnclV7W`g88} z9Ip=vzCde*F17y8IbD}t1wEi(l>6+H zcBkR7;$13Un4B2Ky$=}ewEEuS&u%;3F7jb!`of#%w69+zRv!`ddVCRg4@vbPTL7^@f5J*z!uS6^8R3J_ z_mM0f8rNeTcsH<(ewAJ7N70$vf&fOubm{ z>c_LN&&9cA+7Il!c8fV2xyK6HOsBGX3h$1SpK-1Mj1GZNFEhwUw$Pipv9QKg&iiA+ zubGCV%pG-8nK&mNWf#(}pY%-yjOr5vxw)k1E1fuRtXfj~{yN+pPF_K< zeFC!Hvkt2AwlOVXw5w_4ft6k{0c?^gatF2?B~>82a$aPvsWa0O`mnOOVrWi_P3595 zSH#{dJgvL#-0OpPR`cWrXqr}hXli!upyg5A*DurQcRY)@7HunPfB8Y5b1zc2;{9pK z7=-rJ4nl?D?yg1}gzb}1moMn$spe_&bUXs*Pfiw(rMNWT=2EKrMrtFkOkKjU>K@Zu)BLob=>zK6 zzPK5T@I61~apSJTU;6G%uwqU9U1z+qLx;sDHF0+mBVDz}&tN3VqBu4xNscL@Y2?B? z%f1ezXD6W44ag{+ODh>yHpKqz1ABow;X=s-N{dA}{T`f9;;fsH;IPa^yDcd{8jW4Q zwD@#!1+?vS#!SLMja8z`)=w>i8i5rHND_G-4}Rhdthj^Axc3qSZGh!uE8FO)@aUL` zQXReF8IR?=>!ek43HxQjH3SX&6N`u9Ur?M6SROea$6>WzqjxHQGvs|C(c0oK=pDkF&VFt$(v`o0SLkv%seF@G5hsAzoUQ}aHxCV5BO{VIPk;|KblQJV|M*ddG&aC&#>%?cFsGv>CyG6Rcg1BfjcF4F zB*7icg5N^Bf*0j$FA6>P#MF;IBFr0!6H9)u`Wqr483~-GBhGg;i^730;^r!(f5-@n zp-)aGCTVH8gne43OWPcnwhfLgC{YV<{I?rfPV0_kktN+yh2!*it_O@u;}tkZf08A8 zrebUIMX!2T&K*A%z4dSx^p(gd!Jt5#Ydikp0HEhgC|bVGOY|IR`-$%a(pA1p9LCuS z|D_K54CN>*V>m*7r<bk>@h?96a1 zR2C&8t}Fg}0yx7ivH`nLU!F}IaRVY8Bdpj9Pp!}03UlPy&#`*m&$Xss%uXD#LJvDi zkay$zz@%QV75L?f6hk?B%lw0^IDE9cAE}qxAy)dtoc=Ga-myKisN2?!ZQHh;if!Ar zjVHG4RBW?iqhdR$*tSphTKii2yz9ff=J)|)j^0O`cWZsShOLR^;Jhr8wkvd7C+tfb z`OXWb;GjBKq_^^-Yz|0PwAXn=Wzu0jCXsY7LOZ^1^`$>G=uwkY23qG_f z^s9uBOWLKXC1F{?!pouPmU!nt?32yGVvJI6n78iczQ2Id^v|n@@R*b zw2Fys0EXY7sE&56u2{qs($v22{si~m-td{tX|^ln3$$i>A_rKjf9#XBYVn*N&ejyp zmdFFelz!b@VjqBta0e&!m1}CH{7Np1X(08JEg|d($@2{$`o-coyQ1pFNN>m^r~)J1 z^{iqQT~_!?m5N_-$IGl7IO#$gMk9gfbyX7T1r`C56|cPXEYxg?mIz1JqcTp^lcBIb z>ksB_03G{G{I3=QsjgrNiyskV3(iM31gS&)w2EwUy;`I{Wobicl>5h#LY|#yOYqwxKrwYTiz%(PEJ7n-mn|l+zW_@%lwr%4NYQ6?UyXO89Hcl z@wyH*amNFNHe55qhE~0<3QQVjo9A3Z;h$lRwM~%4sDY&(A?-DkJNVf(M7-&}nK>_k zw9dp4(v#A@S>g_<0(0s-3sBaUzOfb>)6W-#`L+T;KH?(nDX(YpgGzdf2tI$Ya&ebb z`%y*Pb!Gjp>LjXfSU4B<4+8!iH{ZOd-Pqt}7$psQE2~Ff?cmWY3d`v zo(-!wussw?s2OfSZ!yWv^yQ;OR9{B*8277{{%<5We@L`X2szLIOwfTUB5nF@PM{F% zV}??IFFFJcRr=iXDKZt%cw>~kEH7U`=H5|PwM%Z^`y44uda?zSlY}p#O&oCL+K^nZ>c(Dg{+Tu( z`1!xbAnzBvT7w-v!DDVOU(h=)XT?EaZ2?IDt`5ZM2ga*C-fub3Mb+4@9_-d_5Zt%- z(_H$c!0-yHVsu+KV{$ahtFoiqv3TP>J2^OgS+h!fJk~_pSKcdn_sfFtA>L z+){uRygr{&CP6psWnF2VQ!7QEhV9n}l{ai(7kJD3g8U)}0ls%Q{;Iw& zMGR2Hih6M&Sh!E&z;2{+ZwA;WZx}E@SiL>bzRAaum1@=8J&xUCg5n9+lz2W+f3}S1 z&u2}>OBvGt1n$kOC+fN@V{Gjm=#-*;UD)bx?Lf!K^j<-F9er?5aEJGI#61w@89U<3 zCF6$${Ny%^sXz#C#^I4GPFJ0b8Uq&g&Z9s%7{8`HRf``$O1f50t76iLkT?cV%&cpn zuOb|$Pz$P0YJ-R(BxtlQXOA6oyM{s%zo1B_!$-4e3_r`%e@tk;pbsrq-3&7(ngc>> zuF$oG0b}BZ%;oyS8JHl`DjO+~E=R$4E7F0fS_0L@J5&M@AgE*a4q4@M4qnUIhDnsY zm?bhHW%cc%^N+!c?FCExHu%CfGDas*=TiO&yQeRUb&P-s zy_D~YMXVqHCwupT9wIi_!^-YN0W1m8H$p|%Uvvno@6vIc?l>-9c{QBjumtrMyqpQB z4tDQgMLImMqdgoON2m0#y<$&>}!nX>m>0v0oj)7pht zPdy@9g;;9SHr6v9)^vzOI-=iYXSIVgBX@B!cCIr{GzUfnl#qabHTzZt=gcy6HL0m4 z4v26X%l<7xaOA?P#DT&1$ZT&4>)1GW^kx$77P)waK?}J+wIu4Qzs6^-s@=l-jAF;V zE}Swi1X{-R1t&6fbLBPt)h7SUxT7QqRHg3B!T5gMbi)!}7c3m0{&TWobqI7LqlYjw z&oSW(*W>0cs}xj9Qvu4!X`}$+Tc%%vh@&Ba)NlVvJ z#wO#9D*$dWu06mBtPVPEGl|((0 zlbM;UF6PgdpNHszz!O@{x_>DK9sl$@coFUY6AUvh)1Z+W zPS{ROlzqZVbNQ6}wEWEJrygPNXHrf}^^aT(tr6iJpFkLl10(Qh1TE(Le&wJ3slL~&*dgH!VTSucBohgY!?2%ClX z*CR7fIm(&ae0$buT9l~!jU2epUQ<~J`_V(LlXXlSEQSc`$X~qMp=nI@VL$-QOe0LN z6b#sx7Ewio;4L$7I;^wOEK!i1o6){$fr#>@kLZFIe`_>_gw|6Gm-8q4hD+Ny>HF*8 zZYY3LX<><=CDkg_Yy(F+gk+}P!8TwL5c%L~PSQPFxH{_TASbB5qWns!iI_jQKv6L% zShq2YJH~WNyY?mD^BrnyG`~t6leALClR-$Tm-*jN$NxBen~z;RIe(%k5)puaDE>>5 zCYZng0C1Pk`AuYSy!iABSD`uF6>|hlrCPIquId$)O6(`<<`Uw%Jd`E*=wx}&jT`!}-rY$IRV zLxMfk2Pki(p!1DGmH046%E%G9Z*B@B7$aVcu?&(N(F_pw zX{$ff2!2Q*1B+8!5`CEcY&w717>Dp%eCVCw=Zv8mrXIo(#Fra-WJXvnCX<6yW$h+_ zt;z&Dck*ZV!-4cr9%fP_v!{o#I-QL5%`tYBNhxNUAVe_Y;fU!xn!;T~x zYH#KiNg(gr>VR5mkv3bgHjR{|BAAmrgA5;6Z|e&G%1%p~(4-FAiX*Y2NgRS0^J0QI zX&Rvr3O2o_8EDJ+*jgwkS!{6S^UOJbtJ^a5v8s#Ic&cja%z*})%6>~PcKr&wF+Yn* z#&C)SjoY9q2Zt&vd)?~!@pu)BRhH?AplzL1>#~aKW%sRSbjyJEX7>07t@a!<`Z7Q7 zTdjJ0FX=zLrVo2GV0t5M95ke9>NK7zN1{RSpvNKk1*S3h1)azb$Qs|%q*h{pA;Z)3 zMK_{=j;X%uC1Um=)mK59SLvtkAQ+kBv$c!(FZtHSujKx1l z!}LJux6byY@%)@L_)^NP%JM6=zr2vt_zQR)#f=Phukoo`7s zY?XjPAaFZj@e?6hYm zE3W$xx0iNYtm&PVN0PrWevNQEPgjMHQVEGwi|RHpBHliC;lWIxoBWDf_vQc4mvyY$~`O6 zEvz#DNIr~}@|{uuNlPKeYR_wPGCr`r6OBuam_uF*zhl;P-jTElIH#%jw*%ba{snWv z0}~di;qEnm-PF1z`7g3cAuJHIDB{e_o%v^hX9eHlgskW;#m`dB!NHLRka}j2t*H!n z$<9v#sr>3AEgtVc0Ubhh_+1k?FV`GERe&c3DXv>AlGhp!7(!VvMzNB8&u~a4Pdv)L z6-B>f2Wz}Fh3{W`(JTy`&FteCQv0IS6&d9yUc%wABG;HhKP@#P5)?t_XMf0pzB(aT zEy7(ZI+q?FDVXIzs6_yS6lQ-=JKKgM5jBDPP`oMtA`}lQOce*OrC(?}V-quvdLS%<0)v(qfs-PhFTTzogpF1Q&QchPG<>&v z&~Pdb8*G9%;CnM6_>@+GQAnCoVd!p$R4MZqC?RP_K{fOCFj_8I88M*A@2hf+Q;3te z>WB@b!^X>zOKRzCV{XPFQXXW-JLT6O=1;xgh^rL`h*N2dUbT-&P@gId*o0j3#U`JS zwfRu*lBV35nC8Cn3lc-k-G3Kp620f56^hZ#t(lN9nhZahcTM}_#dlr^h&w|G%aifi zeL&o38I&aJ`uaseDRn*<6r>}|s-j$qsBLOEUBkLWwde7RM`jb#YPlxvAa;_U=hf(( z50b9{vLtcRZ^5+gF25l49{BblBOIQ+c*Zk=(aKQQ$jXkkT?S40w4@Lq^J+-{76 zAiS{21k8Rq&!1>GXHuPDH;7=(!39&&2!ASj=5g2V44KY}63_tvGIF8`&%^5NX_qJy zUb`y^N*#$4?C#6CWfd8Tl*iIU8!3xB?moLjF-;DCZ#4_q*054hxL7FnS3x`5OgcV z-C`IR5@<;^S#%L_l$`|noI}Y!Nu5;`{xouA6g@YfF9k8I9g=eKXI_IN2UK5?LF+{G&AUB zl?#eoA=OzdBX2EgHPZ=Wst4APelucyUM+X+XRXU-eK%s)9L%%*_5Cu!GoE&|DIJ%q zFT|nthK;sdJYlo#dGx4)yzR_->_>F#`#8YLoll-FBHMnw|E(n1LfY*0!pguNhe7X( zcsYlShDZC$cJt8Y?`!3e@Tqm3wX3VfT?`d3$)mg8OTSZlm>;wHQT1DoU(C$$?o#i%wqBjubL;t1P4`N+LlpqR z)G(tzL%3u24cMUtt9Y8_A@vo))Mzz;FB2q_SQ0m(h+5_QRgGwLnADad?&1@O;-)TX z;9CRl&5}giF9hSQ`i6V=egIQAA4L3Ef;Xs2Z<6(ab1J`Y()OErXwvort(i$a#Ocn5 zicm}^1i0BN!l-yZ0nR*{DNI1{w<7>aHyrl_TfA+E0>ir!K0`q8+x-_E0~qFBpv$ct z7UmxM#Z{5L%NyDQEE@l${O==qt^zE#NNBXV?%5I4RVQ>ZieOn9p{oIYbV*qV>Y|Yg zU|~cNTm>2-Hf&RIXlDcX!f_&(@`?wbahL>Z^0D7=EOlVi6+#eZ31W+Mk3b6J&RI8bKM>2_bNeQCTenL|(uwHDFSIiPXr)K-ndgA=U(^vI&U5fmu+k&A}#?>*uo)?1SHtPfkDw0uV(Bm zi?rBA5Z7?aSiB1^QBrp*MIl`A55~EWdCEjx_^eC%?SFJ@#R%_b|O4$@TrX zL+a|^W>^PdSxUY9^L{QqW9{XJygh#p>Zm36QG)f=nS!;zHEte}^iyW*dwIFJX1AEx zR=0LIS=eS-ldw({T;bsmWT2k$R^+WOvFUlY^P1^Nd3$sQKIB=m+ghsqBqz{Zp<~T} z@ZXd42tu5Vjd%VDAv$?sGkZDijHTIaNF<4Lqom=V%EypFzC(5ZXps3+V7SOFGP$=J z;rSVjDvG@OIan?u_qZ#1e-F9!jU{}+;>i!%9`idb7EfMeFf$m5#9PI-#p`JE6t(3y zHsfvC2wY6&29+6FwHle9DsN?2CFbmV|J`=bWXoDk38|TLzS$sl&s|ml?+`Z(lxLO8 zqQ78n^foYZR&9d-v?wjJZ<>@gPZCeDcwQ4UZq@4EQJ?ODP|(2T<~O%2M(WcaZ&$6E zn8N$ig*>8bzjfLV5<{ctRXvl5t(>Ohx$q;^Q^Gm>PUL2`vss*SPt8B$dP!H$fds>8 zg@uUC!N&zu+qWUYySfP8nfFVl3ab@TiGkm-9F2*}tjvP|=J6U?T!~GrPeUl_g~iiQ zv{oph-4o`v)^z*b<2)u_!eY`*yJW8jSM2J%i*a*P!1aas-o8yEt6>oneh^U!2L=X# zVta;G*^%^Rqb!covO{n7R+%Q-Efgmh%VjPtonzAA6ec`LXt_32Dx>jo#4q~40~+Ws z34`vlIb_*@N=_hxrr5Sw z2=yD@4GxdahQ%vYxV>z$^}3GCCwhV~RbSklf)PDHqu1|*mRPB7O=}cwy<8IVn=veE zRIOf)-d{-^R@v;A5&l_~7Nn_i&{cpG-qfh;Js-MD9QJ(PY6A4S!E-gdJ{vVW(KeV^ zH~yd^n%ez$GO|nRB?S7qsuy5=rNmJwbOBQ-Gd$?2JjGkmqZek>^X{R85OGGOMy85A zN`w)B6}o^K$0IRmdNI{WTu46iG`XyEQVP`K_aHQ zj+)asILRnfqu@Ywd0F1)L-}|idBT^A%P6xf=ecTV<61~lRjV3h`K8CfY%9py+%07k zC$ZV9p`@d>;Mi5U>C-2)t&=D6a>xDblLvoXJEP9m5JO_Zfa|ET$}^1XzC5a^r|=AL z=RcW>lvBic&r>!*>lxi5So}dR1+&Fte1PX9z1`_Vh0reJ#L=+PwqslzifIi^k(o97 z3Z5}@4duscYF7q|A%HEtx+1@2q4^I*{8oL z(YZ2P3zKcxJJVz!{*Ex?4}QtZ|E|`&bG^p7&@<6w5Y5zPqaFUoBs$ZZYwBLg9nm|o zDc(J|ya0Wc3DU|FL2aZ8gIpDl(&PX>LEVj}G+5u^{1&ay*5+{Pu zyu1o0JY)QdXB|E${f~SSw>FNvPB3@)ql%7BouNp3OjD2*i8O{QYxXE{=`1S|txFkI z?!k~pxa9p*I6%JCq)Ah-4;I>U&WtXuMPHl?qc^65%gTj7VMM3SQ0f9j$y`YwxWpJ+ ze`EnJZKlu&B6fNW=D1oar^Xd9t` zH7YM_j7czT-_e_n^z@JKY8V#nJH*i_;7P^_Dm+xC>~w^QNc}=ms+tQRQW+wXUU((| zrYzT=tVF)htDaTl`nQfR_NOIcs%HcA&CgD6ogHb8Sp?wt%=?xK)-M zBw@)7X(#-Eo|C;pjT0SleuUX$;0xeP(yx|-ny<1<8^7Y5V-PGO9rCL#9|zroBLUmJ zDaE$!K_JK2W{ai|KXwE>*K6A64!8cDJR?VcQWpBA)S&t&|M}a zan(6B=c?mkifINblJpW$ogv6V$cxPOQM!H;zX+z-r6+ePOL_OOlY!79-a}tPGNLh(TAQHNvFt#0!Ozd zGD_)bg$(qy3|(0yU{5Bb+p-qGiKZ}>24b2cnTGUEx%A#5tU^|kZCRqEot!5ejk`cA zU$L$FyXm9+1okv%^9N@nyYxZdyWb%H2{td9MZ*&QAkO$d4=GUcza#SGDOgnG0~(qs z+VM%bmP3&LNo1G&zj5paaM(Yq-^bWZNpMaeKg;i*B|c(07%)sz3u-Fx&r1f~;OCW0 zaEhJGsfU=1zzv&ha0=ISh5_+^Ed955X9)jKxg|q4_enm*R|4?Y(<>?>T}Pl7&^Lo4 z*9QovMx((9N!{$U>YWR(k#C6kg69ut^oa&b8-JADX2v*8+Xou!v|^!r=S)oIIP`Gy zvikUcechq=5vh~0khfOPL+ejF~>pT?W1&Mq!8^AkCMhw zL&U#I(}NYlZ2_dV#>AY%a;9h#W%Xmp9%|^dCMeY8w9;aOjEBa(b(d6Kl8jnc);=6y zWpd~nqMvJ++flvMjVzIaN^)zOjro!nypo#9A7gS1lHt`ZqJ`UjPdGZAl~iD{Rw^-R zc+87QLhN*vQj3!?V#2^^#d0~rsSXryb!VDPz-XutKm$aQYiqI^jY?bP)K<7q;A)=M z-NmGn>-8TfI1g5w$rx?^j8$5dYcuFGoVu9nhsoicnUYd6)WXI0+Dh zY_}op+@}^vUsz&_;YRVQwX^;a_lOC3sw#Zjqilj!Nyrdl1H&pspo1yBZ#_HQ@l-KO7#(f%j&59b#Cy_Ooy zmBBKujI#=f?k>%BMQfOFej&72FxN#_{Fi!`Wer*KZdsy@)-MM@rli^)Api6GF2NQp zN{%HK0{r)-|J#SY+nK((oamPZSRk!qKorS{=nXL$x?F;|r%_|G1$XyV=4HVP@&`jt zf++di5aPdZmTCPN=lLX1ldOyB$>}e1a}WFX`;|6AAakdUfH);ks~ERrw%Q^~qDU!F zm3mj%^9-Xg)h_ScXzOugjzPuoWcDfwuO&cBP1L!j!XMk!r4qk<@UTUE39c`_u*3$> z=UWyL=kY2OzA`x4jYm11U;Eh%TJkAfFdh@SJkp4^4j3apm+e4om>D z_Uej*bqs88K4hVj46_rPzfc`iiuNHg_dyxW5reA*z8&vFW zZwnJ$bxkyX4d9QOSOoo=buV2K)Z_F`p@$o(Y@?@r5|MZu7R;VWsbq?VYqW096XS%# z=fpZPIknkqx!I0H(BSl3O^RYJ+cyVbv@y*vc~wK0W^hffu}%{7vY*_?3aEw6wc_$$ z^uRdz!x8eZzX2>WED67CI_#WI?+=&Gg7FRRSUlP;Mf3w$U>1Rv3)^k#q3 z;!DDtC5o-sEM!|Dr|A5JjMG)&yV;&dgn09dqH6{XamNzvexJti$%6zI*T(?-%Vr6M zk1f^pwf9V-Q%P`FqoG7>nAsY05U_%|Fj)u|5TfEXJ;+v~k1WVk!}RhIfM|j1XI<@p zU5FK>j-}=$dje#Ly_u`m6c_h2iYNUl3)ELJ<|iYIt((WhlXEme4G_GZ$0ddpGKcx= zw?MIL#aLupxTxX4M@x%ou@JaeaG>4wOR_;^J1BEUe%FC$$@0j0Ot)=q_4P|gZ9RGo>D1XqJI!&QofNcHrcCIGD-T8xR{#| zInK%GOXUqEadZnkAn|=VDcBikEVEz1yZD}HSWlAn#w+p%Ou4%=e=gRV-Y?7et3L1Z z4g5iw!oykQO>V*9Wmv-M#}bo?jkLru(g)jvs_+P-reH=Z6+0 z^ZFdJgMS7bUFHow-QYF4whGg;*G&_atL<`kQ;=-gtj)H1%F?RHB9G|hYV0dpiJ#;O z$o{dw)tn*H>+~9}0^G`0sW|OesM_RI+Eyy9yK9rz3mL~eHAK*?GR4`=d%JZxc-Dtj3NMT*tsR^~4M4TD0$e2gF8p=lNCu~I$7#Qj z%(pgS`Erp5tFex-_PRGs5~9@ym#Hj7g%LbXWMQw+bA}^=jB<9{M-zTwlbF(I&K{MFf)!uH zYp=ieOO7$tVNropaCRV-*63SCTstre6Db-VI%1GefHbh6TYV&3C#3vc4)@;kk-xZ9 z{59a68uFxdqy^6M69PN?lgyqJoRgOkHxFI?agl)nrKm#0iq6?#Lg3PrzLw`%+FLM{}bXNamO~4C7&^ zF%QhrfY#mG^0cIwUC!0Xx@nj=({Nw;>h-6S|27t$VgJPTVM0gZQgqo!gh9`4r zNV%MB)?$2d$a4Yy*+2L6N&85^@ONJ{W5$c2?-L>R?f&!?M6j66p2+d^CinBEkGzEd z;QI!-kAefXUWwXpl8$?meS#lXhN9cH$lPRjgoe1c#24KTm1SZ{Pq-D?mE5SX#P`*A zb|=|e0&wA#L){h>cjF$aew0JeG)Ie$a)e8gA}5gIUrypiwvnjrUx)mF7>c3v8(Mzh zxy0U+1dbe&&kDTb_^J8Cd;Ze%ubHjEi{8Rx$!x)5DsqZj&{m`Z-^jf4xgZ|&3abB% zT*Bk-Pv9_nzfmy=VDJPRBuCGov9Wf2hZ5;W+uuk$>GG#QP?5iIlaPutiOezxCJMpI@+g)Pzpx)+aFU{8MmA+rsAg{;<; zJr1i=8B6Wm^d2^z?x5RIrW^&&;oiK-l@WmS@;l&~1uwcKRbi~=?i%cMzc+%4;!@fW z@<=%;9Is{(xMMXi6U>pr2H~e5KV#Dky=-X4?BnZ39=EMt(RgGI%Yib+euDjH7v+1Q zJUt8XLoXmF?+Q^Sx1ZwyrZi7IP=DrWbaPA)nL#v zz>Oy$6tlXS&j;aNbw)%G>>t;5W zBnN!lKehpB--gC`Z;}@Qd&sm)h*NW54#f zUQ(hA>p>{!#Bq|8qJ3T$(*FG|A8`aDWiei*F~Pms&we-CjDf_u?S~>T9(#h>3<~%1 z5*;Y={lqd5&<~`n*BeNdeM<@XAstA6Yl_qwWp>xvWk^BKAe&x-LHW67mD+n0vyIN- z2c|Jh$hVT&(I?IWboQu=N(OQ87AHm1jm|i&T8lQKDd!ESk)Ft*=y>2O6hv0hZ}IAZ zA}L9j0}~|w;%T_JLC>JJ)Cn_!f@MXY^8n|o;V&y&wKg`|G;vh76zh>L&6{yCY^6Tq zNU69ia4_rh9eS61=gv=>${2d!HtASpgl$kb{`L>kr@g8G?8Rkf-opEV6d=$FE$0qX z@t^j`cKB7bGK(ctGSJqRys>Tky9UB?hJngF&MppBYbEv!`4L z?l^_f)yco!4$BPVC^UOjJ8VMD5R15_-&$8R+_H@eHSgqqQ%_e#*3=d-$!tpQ@CiOT zprfvFXdh$+#8HJS^Y~Ou%5eYHoxq>HY!RxU64B>k&G$M@#=In=CF>$G3J<{q_kaIB z)^)lt)7MY4)%!caqOa|xH9Y}SJ80!d6#lGj#anCO;AmKN@0i7nkER=$+I4}JkzIA= zzOmfu6nVcS1H&+E22*IMJ{wd5lUAVC@2p$+PrMo#5Rt2$D~nRSjgVq(tjVR2|rq( zdv_lSkVVk$?+S8U_9Aq;Y?u~R0)M|P4QWH`iuFM2ir#`j_{R@^?LxNRu|U1@8)u@? zeGz$Uxoql#=$GORvCcR0ue<}dUR^Zb0fcX~e5n^*L3(vQ7=+#Wz7mU7RB-Jwt*PXQ zBeVeasd9QKw&~(haO<*_%5wPQ%IbwByQvmi0M5%WtoCnvoSLa?yhTSVc$I2M_Osq} z%nY*g&Qc6C_;e6ZWBr1k|6wW}nXZt0TDgn&l)Z3J)#NBD<*de*7^-s-mI@`+i5Drq z6KSxgX=-;CN>d%m`rfRbUslaWXH;aUb8>;MR`J1Qzv1PMH`qWK>HBkUZ3@t;}?An&lbL*|L ze4PbL_j2K!jLKqHdJfH1M3v>VE&RTB2V}Wz%ex?{D!4NAqeCr7?ot)R_F@E zX+Xq`_|zoX9$E@%8aR%zaiss3tEsY^tGR-5J+z+YO}We|`i$SsFndLxG@!{p2Vfra zY50dl^H!BI%bBrcfb)s6yIsfhPBs0;z!CO$@-gSbh9`Y^L`f~kZOHP9cu(_8OG#uq zz!NixOE9Z9UfT)DZcoHDDxMFv((hCY*1H~NCE32}6i;{@9QM=~ln2y~({mkEMlz2r ztVx17pFBRxY|JU)XDS+I(>2=RVHCjY*7&Fgs*a8ho|8f^5HQ>P&y0~S2 zS~5{^opNn81Y17HIfew7v>y>XU*Gaz1F2uW-a$f%Q%G$m@fgFEEE*PYs3op!)8X3) z{*Z?==(TMmaw0eaeryrcSnsGfGJbwyk4M=lb-~A@OLfDK@i>$<;yBk20DRxA(Gis8 z{N2sX?ytHu3WaL9`qvAXZ{Ry#RBy!F<2s6RKtkP7Q3dqz?@7c&g%>%LfrOt6^x`}}n8`JaK z(zF^jN_^x}f9|G&&?6xRrTY10JCb4s-e~CG@5C#OS%0U9?@Gs6`JL{`9+sadbRzgK z&Y}J$;#YzHtbSxeJ%uGd?bgFj1?2nD3IETa#^g^Ho*zwNK@n~A+XHuN86BG@B3@Q> z4YUzHk+>XsIvk5K%aEyG19N#Y>k7kxx9~RB;bGxL7$tNF(;rA6<#w}m5?BUeBIzn; z!^Qu&;WEFs*B>-vI4KUynI~zCQi_JkOZ|6dsBaZUfJ6d>wa51T|9$A7&o~690nqlVAo6@?-=O^lS z9pqO<<#d+811DXq2emSC&Z*I~^`tpc4%SL+h##gneFMTZhlMGg85Erkt35&n;>wUO z4j>lx`jr-8^hWk+mcj1@icR6&7Ho8eH3*H9Tfg}UKL2UuuvQAA@;A>@$F&!|rj(j5 zOqS8Y^yD(N_!);^${L5vDxw^EZk)+VK{9$4Kr{1%G>+*{(jj6UGAVSVbhkMG>%+5<3Sy+DE85|X37s8Asyu}41Vs4R+_QBcwr=&$t>eKTJM|yWriJ8Kpyjjl z8Gb<)c4AS**fR}LOHon_-H4>Oocxy~%f3WOfA-M{3;)N)%g>w?tXj@0O4=mkXJVl)o8qsfLG^njsmE|jr~Stc5f6;CB{YYI`+w+xq29dKaNxDXM~d| zB8xs&fBs|=oFM4F|4ukDn`rIgg+sjWzcGBhg%L4bs!(c_I$gD7uPXH%se-?QOTi1~ zq|w!!e*)wpXEKPqjkj8GKmA3(?9pD^Cm3#V+u#_s4(_a#Y7gzBbLq}J(NmH8YPC)x z@A88iRP1?(>OBgDXgd#Wq@?TYVf)Gq>Shy&Q zZFtOZct=;Bke9(a$|8?-T{*R?C?}3g30ufmm<7bE;Mf~$_IHKpXe!p18$nONxddib zciqS~m_x77wpIFP7}cliRBoAP@DLpPcb5lhmr-+DJ{>8ns)UULDr$l{;Y|8n^0Mt_ zoF<(SJ940Le9v&|mI%6T!Rl_rCq5=j)jBFV7Atl)nPgX{rlw};3HV34Q&2k}4X&=8 zYyeH$CGO6(woD931v4e$g!4!chrXZ{DLI~ZA;<;9I6#BgjnW}d3t97&?)|P$=)``` zn7ayJTC-YFO?0XpL1Ys~5x!PYbJ*H5pqIIeP{_yNaPO1RrZTFk`7c3Z;UYa$!~J*o zd~+gsR1AA4!`KX@`}Y^W>tbon@{uVje*v`DI7N4eyCKa~v{{5RN4H7;guj$JR1$7D z(6XzI_mv`R@^T;I?3bSp<=ypcv-pwE9fLhT_fGnu=|tHjn{f0wIk4z@(A8P2VHs2C>Jj z@bVng40Ri5RLn_ENB+J;|9I}9o2TtrHaI<(JDqcv4!J0}dT;7_ap530gXZkxh_1q5o%hTHN z%Q5PgYuR|>PM?G9I6e0)iH@rGxCtT++YQHwv@0(N;yj^*N!&$5$Z$qU*jZhn?0ZDp-0@a8G=m(TRRo^~x-$cOlI-A#`;iIC!vYJdA zuJx3MTxUV34cIv5I)=wHVr)m9n|DeS6g|-Ba&hr(9@#b{lk5)I)T|CGZtK;LgRAm; zU2cMt{!yWXa}TybYEgO0>W|xg$vCgTdZK@x%EXwl;0Q!s(7&8b;gd>|OvwN$tQL~KWaNRw-p`Jd?_oys zhe!d)o5bUu(S`pIMbk+Bu=qqBaEG*Az*Q)m{0V4Rxp1t3J(hIo^stxZ3UzhPtRa*l z`HCZc0`+=g9qYhjk?#h-MKa2M<~SGq%yH)W9}x$UUlJ6?53fm^-xK)% zXYVK+c^0cm-u)#A_?G{H7r&mo5C8sHq`*X#kfWO=CX|NqiEwOWW1uLgAii~K)15uL zZLScC;`dNr7|B2)DSic97ymdukZhBMlc%pHr+x3ICa*r5D|sh zh^@Y|slxNl+n-(9vsJgHtZAW0E3{;UrRnZxVOG|28shL$dK@7eIMwY&&>`%Az2rF& zixs1mSr0982D{@i5H(sfR-j_T4P~QFf1ozgU^~-7Bh)?pSt)s&6`qKCBtN18e42$S z;8GMeDo_Ww;W_-1!RSiU44^-H3fJf&KDBJ0Nr(zQY6ydbW1La-@bEWV!#!N6MlvHl z!g98VG&$LCy%bfdx#E5iHEd#uNa@ z%$N#CXP9Vo0N54h-6s}U!pptur?4iKwyl+A)T#gE%z*|hDz;}jhfts)k}vHu$up@)zxX{y{FTEStidTK9mLTZkR zqAjHV7exHf7wu=lt8S1yYk>j9Bj$f@c?%(@#+2kuA!)$O|Cy5bW$wNaY6gp5K%*^K zJ*#X=O%#%XB#D|tp(blB`M{APr={yYewFr8*EgP>uHPkC7&%u3NM`O?tEIxIxt;YA zILdzRd3H1J_4WBi?juomeJ~LZXTXH_`q{s#ljDbpVP1damKW=P7Yrrnw(0Ty&|!C> zdJrvpmjE3&5%!Iiywx_#_c#M<=sooNToqxE?tKYs+z56vHQ_y%9JJeW z4L4#ZuW;f^ZuGr-5k&P+ZR+L`VOB+A^}Kvp8k?tPj?c6YpVQcu^I|PPJEZC zBfLK0>ZEiY7BFh?(a7z`^(BodIlcOZW1p~D1-0pM(+b=L=j_ z={g_!UL(5==Vn=rPA{*qfWVYMaylE&$@;Zq6EQILC3rf{dXe{`vmRtrex%0z;y@)+ zEWmv4y}9P+46JL3rUYGB;`yR+#@VOBEV&b18#aOHOql(;X&tNPZ;!HIqX7L17>3Xi z%LI~SI@0WnZXkdJ!YERWROSK9`~faLkiB6DUGm*fxPV9jU%eV%cwtXqX-{aS?c^s; zP5GM2NzoZLN>?s_@)b3T^*iVy<^@P337{a^j;2sFcJ#CfIIoXLfn6$gRxl(BM8=n&`C-rmt8*zu!U z^dJ{c5(j4CpO<=$*SSP+9SzMiR_Jh7%6&;mZhNW2l;(|vU5E_kjTHn!>JQ< zN8}n!Qm!R{2M^>VJv-t{seeZ_T>s~GHI(yLOP&-J1nfA>E24eLQ=`GU1Z*sFqa$-- z;VyM!$55ci$&Q%^tqpE7%+@m;E^oSve^VOx&J96+1ASBN|Dn&`A`;4KK9=ac&%Mss z`1-oP#|!{%wlh^~N)(}wALTgnG&UL)4o>jdm16Xk8e_{E1#7~M(H*d!Nxs^BHT_qK zX#CHm6fh4PHXwlUcj1=JO1w;8lBx@n##K!K1(`DXy4Y211=Hx{Zo-{W(Zr&9Xv`zG z;F^2OkOkSfn3v4$QPnG4F)M-*G@TS3-!G<15M>#+oAp+Tt0%#vjF4I(j&}-GcpH|H z2)kfJ`w%X!9%T$IJCvdz=S9Ocf6|Jj>~hb^9MG;MLYxpD*oE}&z0D_cO`SkJhRfYp zFSBYzcjUD^Ct>I7FJ1nPm}U4J?85H4UQDAG7 zU*9%q%eUTpi3dNuX(Y;}>3!JTZK*;}h!e_3Qm7(r%;+J$*Ca3pLn&?_1zoG3e z=y@7*YyF|hFCc6gjByob7h)m>GF5&h85|H{+% z!1oV}HN!sNxA(bcPOQa_A;1q?(~eFzGJJp0V~!lLoi%3u?{$ zV@G6?DG~Y^-(ZFVl=;wfGpS=!Qs*d0bJ%#6a+i~5YqDt=i-~je97!{Rv^3T}A6k5u zC6^@)D=BL!r63_#Rw;Bl+EauFn4HUH0h~u=N?CH&;yBd!;`4j>Xup`&Md%1Bs0_Z# z`lIX7GYMJe(YzQcw)a_jT@Em`P5e?U-ZM$|G1(Zy{4s90x?>%8`4T>VLSavmXTd9~ z=A^tHJ^rMes)GGK&)0_tNHgcdW(vzPEonG*A2AI|X3Y(&x4;Zq+@spespi5pQL3(y zJFM+3sO!E$qrYbQntP>}y_rv$BqfM%txRY92qjWkqXDq^s?Z0=m72~QiP&~xFKW2L z?hnhTF_MQ=@PuxC#_RM8wvY#${d)TR?90W71(SmYJP9lIQrN~$DoeK0+SNT@turCS+4c%ol0tCPXyY=a9o8|vh@ zBoC`J8^r5a2+rQJ8MTtWH6n`%mawzv?@yR`Y|@MAEbPwgM2J2foSRYC#_xab|f711Wzx#U z7mj8A%9_ghsbv*bXd%A>A#AA{|0GM0`_c11Ib15^dfrVj#_7J4QXl%F8bb6`xJ`9( zr@-hSQc97U6tNP$0B0%;v!Rn0^}mr2#)nd-*oQH{v(9+mL+yMpNWhM3GAnOP>SW5X z?Z9Yzd5t_1a8`ei7;8CM)9>?nRrYdVVwe7XJ!~`%p4HSKX^9qpt#Zke$&EIl_MS^@ z_n+HASPdwkGI5qLf2`WTVq09DffFlgTm1T9=8w}WFOMi?6}<_?w7GeGzV*4;y_HFeg0V1|V`qwb zuX9KLopqd-aBR!oLcETcrrPNNx{P*Mi=BYLenQkA^4|@N4<+Lx1qTu|^oF+rLL@Z_ z;_Vu4xM(5nCbKOREPK^&tzszrS}i{B zimpz-o9Bn5csTZZIi!@Ys==^ey&0k1pn8%yevqTGb`&YEQZSn5Mxs~Zy29|5)>SK} z!0yG%f99(?;Ln8U;?MX{6U-DyeQMeKBiEN}5VupfCY*%}SrKA6=TW1iEJC_L__C#f zH%kHWq1N9i7g?=05IGTU<(yL^%s*u-qvb+L9F@zU@4$N<_6?06jN5iAr)(rL!09S` z&d6D!mQCagdo^^I7RLT<=5j|FLv^34wpQ)itdMYu z+k?*f{e134e(!^{$q7VmMq6zw4p5W zpa{kGAiWJ|W)C4~@JHNwT&>6B!{D)G@Sut3(83|rR{(VdIg{Fv0?QMdI{&cqaboU3>M;wB&O|vvztGJ9+4MKEaf0Y*k4%g!eaRW4#_UOnS&2?CRRi^qNCGzI#D1%8b?C>Q+Fw#?Oh+4Hh*kGhB*!9r0 z%4mE7k&J&OKnX3C3K!v|-Fr-LD`qY}gH2|*;(nW^WXN+bHZ|@eY#;B6Ps_6mtraoG zOvT5%8ybLFr1mv|+vRp(=m3TVuOq^%rKa(YXr`=vNYO}3N5v3(W{+Tc*4`h^T60+H zY5D72_2P-OWnPq~z3+(1)9Dj)bnD3o$NfwdSy5l?q06wL`=YTzemhc*nytr0EzkN$ zO1;Cl*qk|ZuDW1(ROQ9maYlJYlf~3T`%{m3oGxq`-A$2<6u~7PC31LYXxoJ7oUVTT zAlbyA!9#Z@cr4@Pvk79C0K64Gk~?*A=vWb5{bs8$r#otjRZUO(X*z|4)=B)pLoK0x z)c?Z+U~SBPygo5och2N>x>4Tru;*;!m*i)G}lEkURZX^R$5hOEM-SR1Y4JPT}AkY1yrt)r8Ng?$qV;8NpxHAFQ&&$B|H;Fcr-Br`9_Bsk&DS)_&RI_VuiJ>V!?#uU6L`1Q#5pFx`5O=WWC=KEb=)TNfJq3kT`q zE)gv+C`RBX(z(vt9%$~=dJ(Zj7X)sQ*%)Ylm9S_;Z!6*MXv=eOA}|-N8Kr%wHaqEx zKJ}FbeLwu#B_KkU`EEwQ;*nT~+i}#y3{hk4hUL;8gP|QpCuW$%=dSMMGCV;am2i|7 z>)h2=67-&Ng;8}wkyM~xb8DQ=Q`5K8p{4bxQlILQT#7}P%ryGNM6&gOPA>=goJ);& zEd>-AP2b8b03N)h;|L1${sJkno}%BMCo6*~D5rqqB5Y3bFQ>n@!4w@acjWXEyiCqN4I9qW3`~ za@SYKqXw*JT&>Mac@&E$pGrOt$+@Z0av&maL@X~!vQ|RKVJ|klkMrnK)gojs4XXKw z>Aew&i4B~tlN=!uesyPJGQ@C^u|cl)u!JBr6T54eoyn-zt)hp`BKDD!44V*2-a3{g zz?8&A=4Qr7Aa^Z96sY6UUQMo4iUD>(j6l}ul{YQkz zlga^|eaBBC<_|+O*;W}GJU^q4EvIrO`vmFNmb5n(yWfd@s!=%TlI@h~%cri)a$IpayA~I+G3Bk7)MiEHfj&k!baX;dSh>Vm%q`q|S zura4lzr_iHIA@s#`J%0Yv)XqJHzKuNjgV$wKI}eTVgDC=hrR>z^mm01PpkMyM`+*y z97T+eBcBViE+|r5*W|sEQY!g8w#q7Lx&Xb5Va6XVph1_vvo!7&+8~*7CB9ekqjqIe z$YaHnD1Wj3!4{-dW7xM(>U+i!X{95V8N!O4ZskfbAk-vz24`GP{L6f`Qd-p@-{)^G zD=g#cXy%-A}# zuF2+Vy&^43y03AKm`_*M;Koo?nL#*>^#s}RLl3&Q(LSFDl8)`j12@~2d{G%KeO-r_ zPXf*9+XZI=*+)4_d6+)EY`mf~FRSPVamQ>VT0tzcAIFE`IJXt3-r?BvWT)cXWXX%l zOx9RZf2-kjjey^vGw6{eP5(&DBkjV5cR~E*^_5?H$O2c~Q@TDZ>s7vEwpi;BqT(xN zv>f`%_xia-ZIhk9O|*c2!_=Zo1@y(I{<{rx!4L5bdb?FWM|Zj5-g++wg_W1kTg9bH zESm=)Rx1IhgMj0bcjqia=hRcE;a_1ZXv~zZt|Y;JBSBus56VPQnaw0tIVeC|*!-Pj++7iJ4?#^AOyG_1bqyKOY&oeZ^I^K6Q~I>?6@F z0v-qc7aqxev7B}MSyO#9XnXp$`!eD6de37K!Tt$&p0E_}aKA#;Dm?$Jea^ay*Akz(0Ou6Z#p4ocICJ;$y~T75NIZll+#wP)}k99*yY$ z!UV<%cMS;aTGIETbR_+-wnkE>mg3dLpYs(j)FM28b-LiTkB7VQ+#s`*mcS4mpKiMSgj;C)|^P zNNwCb!g7mvRDm=TgbX-9lB*8n){H;h{!7;LQz*g_E!tkLmU8#uM`e2p72EKfZb$Gg zTWyu4c=h$ld$a4&O1|FKHrx%}e~LbRFM&m5L>+n?n}@4se|%&ArZM}|%KZ$l7Z2Ve zq7)npgIqhpzvo(NjO}6uP_c7P$~6nIsAu16H(5pH$vhhJ(BO}@a&JaTilP}h7nnY> z(3dO!B`SJ|Yn7%nj1*sRDAErX=j{_Sk3dK)eSPWfan~8oWY~C?-JmF zmtjStHpV<}m&-lKqHC_2JAx_K%tuiS5te88)`L`1gAdM}Lw>ZgeMGckR%BL8s91aV zaJZnlC8Qwb&Fv zyQ)FLX0U_${&!qTIh{T}4wy#Punk@kYl?RX%r{fAc>g2c%o`0Y7kN^P$^7p>p}tRw zXOMvoRTA*`N1of6Q`MBq#M%--FOz|}Zn)P`|0matj1YDaEU`k3dkIX1!-SAt!UHhz z$pAGJ114S+Hx!6Kv4XaLi26}9KR=zKn0|f|Smroal_LiBKD!oldO<;_7Z2dR%nS_f zlJYV0a+_EiJ39VU|4*E=B_`(^dTb1gAI}3`=kU)f%a`dAU^GGol5c=!ws)Bz;W%f4 z7tgM7{;uUl4sVVy7%4poRsxauM+XK*maiS|PZBQR;Q{*(aXsc67Zh|#%Y$UT3q(A^ ze~4FnDE~u@X4>u_Cc?lFx`BbgaDnK2?I#h3;KCyVMte+Ubva#f>sv^P%F)EBhr4r1kuzu>+tCT@i|}U z-=*&856K{xw9lI9`Un86L%MUOE>v8*U}{bU61NS4HRSsO;&0u{J;a{Bx-?Mxn$TRr0wHxQMU(fRM#Ma0m*}*?sv4Lg=0nX=`MV_^>lzAv5)VhV(*S#0A^ikpRQG;<;}A8_|w|A$kFt2At`yS!X;4|2+io z7bDBJFX0SM1*ZnYHZGr|{0Otk2X~KlFr?+8h2CTUbBhfCHfYYle~*Cq*TPWR8T@pE z{2cs3^uRAH_@3qaDf0|YE_xkE+dAK&vls%fLsj683co;^c@0u{fT2zNa~95=wIJLL zgkS9cWG)DoJO};ZD8mAwHQ+{h`-0{!FlYgsA}_Wm&cT1UPG&*46}TabUw~5=ox!)a zn9srg^`n3Z2p8b$r68OU(A~xdXCx)d4cknRhCWch2jY}0pa2gV z4>C6h2&3=WURl0V b7hw`Bw4~VJY{0;{1OBXmt|7YjKmPh31Up)j diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index db9a6b82..1af9e093 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index fbd7c515..1aa94a42 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ -#!/usr/bin/env sh +#!/bin/sh # -# Copyright 2015 the original author or authors. +# Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,67 +17,99 @@ # ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar @@ -87,9 +119,9 @@ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -98,88 +130,120 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi -fi - -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi - -# For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi - # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" - fi - i=`expr $i + 1` - done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat index a9f778a7..6689b85b 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -14,7 +14,7 @@ @rem limitations under the License. @rem -@if "%DEBUG%" == "" @echo off +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -25,7 +25,8 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -40,7 +41,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init +if %ERRORLEVEL% equ 0 goto execute echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. @@ -54,7 +55,7 @@ goto fail set JAVA_HOME=%JAVA_HOME:"=% set JAVA_EXE=%JAVA_HOME%/bin/java.exe -if exist "%JAVA_EXE%" goto init +if exist "%JAVA_EXE%" goto execute echo. echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% @@ -64,21 +65,6 @@ echo location of your Java installation. goto fail -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - :execute @rem Setup the command line @@ -86,17 +72,19 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal From 2e5ef70fbddb4332a7755b06578dabc6cba07d0d Mon Sep 17 00:00:00 2001 From: X <24619207+Undid-Iridium@users.noreply.github.com> Date: Sat, 9 Dec 2023 08:19:36 -0500 Subject: [PATCH 435/512] Patch/Bug fix for ContextMenuMouseListener Actions (#161) * Should fix the issue associated with copy/paste. * Fix build gradle, and also create junit. --- build.gradle.kts | 4 +- .../ripme/ui/ContextMenuMouseListener.java | 95 +++++++-- .../com/rarchives/ripme/ui/MainWindow.java | 56 ++++-- .../uiUtils/ContextActionProtections.java | 30 +++ .../tst/ripper/rippers/ChanRipperTest.java | 21 +- .../ripme/ui/UIContextMenuTests.java | 187 ++++++++++++++++++ 6 files changed, 368 insertions(+), 25 deletions(-) create mode 100644 src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java create mode 100644 src/test/java/com/rarchives/ripme/ui/UIContextMenuTests.java diff --git a/build.gradle.kts b/build.gradle.kts index a5c77e48..e9b1ce55 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -135,12 +135,14 @@ tasks.withType().configureEach { isReproducibleFileOrder = true } +println("Build directory: ${file(layout.buildDirectory)}") + tasks.jacocoTestReport { dependsOn(tasks.test) // tests are required to run before generating the report reports { xml.required.set(false) csv.required.set(false) - html.outputLocation.set(file("${layout.buildDirectory}/jacocoHtml")) + html.outputLocation.set(file("${file(layout.buildDirectory)}/jacocoHtml")) } } diff --git a/src/main/java/com/rarchives/ripme/ui/ContextMenuMouseListener.java b/src/main/java/com/rarchives/ripme/ui/ContextMenuMouseListener.java index ad75ede5..dac3d0a4 100644 --- a/src/main/java/com/rarchives/ripme/ui/ContextMenuMouseListener.java +++ b/src/main/java/com/rarchives/ripme/ui/ContextMenuMouseListener.java @@ -1,15 +1,16 @@ package com.rarchives.ripme.ui; -import java.awt.Toolkit; -import java.awt.datatransfer.DataFlavor; -import java.awt.event.ActionEvent; -import java.awt.event.InputEvent; -import java.awt.event.MouseAdapter; -import java.awt.event.MouseEvent; +import com.rarchives.ripme.uiUtils.ContextActionProtections; -import javax.swing.AbstractAction; -import javax.swing.Action; -import javax.swing.JPopupMenu; +import java.awt.Toolkit; +import java.awt.datatransfer.Clipboard; +import java.awt.datatransfer.DataFlavor; +import java.awt.datatransfer.Transferable; +import java.awt.datatransfer.UnsupportedFlavorException; +import java.awt.event.*; +import java.io.IOException; + +import javax.swing.*; import javax.swing.text.JTextComponent; /** @@ -20,27 +21,72 @@ import javax.swing.text.JTextComponent; public class ContextMenuMouseListener extends MouseAdapter { private JPopupMenu popup = new JPopupMenu(); + public String getDebugSavedString() { + return debugSavedString; + } + + private String debugSavedString; + + public Action getCutAction() { + return cutAction; + } + private Action cutAction; private Action copyAction; private Action pasteAction; + + public Action getCopyAction() { + return copyAction; + } + + public Action getPasteAction() { + return pasteAction; + } + + public Action getUndoAction() { + return undoAction; + } + + public Action getSelectAllAction() { + return selectAllAction; + } + private Action undoAction; private Action selectAllAction; + public JTextComponent getTextComponent() { + return textComponent; + } + private JTextComponent textComponent; + + public String getSavedString() { + return savedString; + } + private String savedString = ""; private Actions lastActionSelected; private enum Actions { UNDO, CUT, COPY, PASTE, SELECT_ALL } + @SuppressWarnings("serial") - public ContextMenuMouseListener() { + public ContextMenuMouseListener(JTextField ripTextfield) { + this.textComponent = ripTextfield; + + //Add protection for cntl+v + + generate_popup(); + } + + private void generate_popup() { undoAction = new AbstractAction("Undo") { @Override public void actionPerformed(ActionEvent ae) { textComponent.setText(""); textComponent.replaceSelection(savedString); - + debugSavedString = textComponent.getText(); lastActionSelected = Actions.UNDO; } }; @@ -54,6 +100,7 @@ public class ContextMenuMouseListener extends MouseAdapter { public void actionPerformed(ActionEvent ae) { lastActionSelected = Actions.CUT; savedString = textComponent.getText(); + debugSavedString = savedString; textComponent.cut(); } }; @@ -65,6 +112,7 @@ public class ContextMenuMouseListener extends MouseAdapter { @Override public void actionPerformed(ActionEvent ae) { lastActionSelected = Actions.COPY; + debugSavedString = textComponent.getText(); textComponent.copy(); } }; @@ -77,7 +125,8 @@ public class ContextMenuMouseListener extends MouseAdapter { public void actionPerformed(ActionEvent ae) { lastActionSelected = Actions.PASTE; savedString = textComponent.getText(); - textComponent.paste(); + debugSavedString = savedString; + ContextActionProtections.pasteFromClipboard(textComponent); } }; @@ -89,6 +138,7 @@ public class ContextMenuMouseListener extends MouseAdapter { @Override public void actionPerformed(ActionEvent ae) { lastActionSelected = Actions.SELECT_ALL; + debugSavedString = textComponent.getText(); textComponent.selectAll(); } }; @@ -96,6 +146,27 @@ public class ContextMenuMouseListener extends MouseAdapter { popup.add(selectAllAction); } + + @Override + public void mousePressed(MouseEvent e) { + showPopup(e); + } + + @Override + public void mouseReleased(MouseEvent e) { + showPopup(e); + } + + private void showPopup(MouseEvent e) { + if (e.isPopupTrigger()) { + if(this.popup == null) { + popup = new JPopupMenu(); + generate_popup(); + } + popup.show(e.getComponent(), e.getX(), e.getY()); + } + } + @Override public void mouseClicked(MouseEvent e) { if (e.getModifiersEx() == InputEvent.BUTTON3_DOWN_MASK) { diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 6cd5c574..d2ff250a 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1,6 +1,7 @@ package com.rarchives.ripme.ui; import com.rarchives.ripme.ripper.AbstractRipper; +import com.rarchives.ripme.uiUtils.ContextActionProtections; import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; import org.apache.logging.log4j.Level; @@ -18,18 +19,10 @@ import javax.swing.event.DocumentListener; import javax.swing.event.ListDataEvent; import javax.swing.event.ListDataListener; import javax.swing.table.AbstractTableModel; -import javax.swing.text.BadLocationException; -import javax.swing.text.SimpleAttributeSet; -import javax.swing.text.StyleConstants; -import javax.swing.text.StyledDocument; +import javax.swing.text.*; import java.awt.*; import java.awt.TrayIcon.MessageType; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; -import java.awt.event.MouseAdapter; -import java.awt.event.MouseEvent; -import java.awt.event.WindowAdapter; -import java.awt.event.WindowEvent; +import java.awt.event.*; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; @@ -57,6 +50,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { private boolean isRipping = false; // Flag to indicate if we're ripping something private static JFrame mainFrame; + private static JTextField ripTextfield; private static JButton ripButton, stopButton; @@ -281,7 +275,47 @@ public final class MainWindow implements Runnable, RipStatusHandler { } ripTextfield = new JTextField("", 20); - ripTextfield.addMouseListener(new ContextMenuMouseListener()); + ripTextfield.addMouseListener(new ContextMenuMouseListener(ripTextfield)); + + //Add keyboard protection of cntl + v for pasting. + ripTextfield.addKeyListener(new KeyAdapter() { + @Override + public void keyTyped(KeyEvent e) { + if (e.getKeyChar() == 22) { // ASCII code for Ctrl+V + ContextActionProtections.pasteFromClipboard(ripTextfield); + } + } + }); + + /* + Alternatively, just set this, and use + ((AbstractDocument) ripTextfield.getDocument()).setDocumentFilter(new LengthLimitDocumentFilter(256)); + private static class LengthLimitDocumentFilter extends DocumentFilter { + private final int maxLength; + + public LengthLimitDocumentFilter(int maxLength) { + this.maxLength = maxLength; + } + + @Override + public void insertString(FilterBypass fb, int offset, String string, AttributeSet attr) throws BadLocationException { + // if ((fb.getDocument().getLength() + string.length()) <= maxLength) { + super.insertString(fb, offset, string.substring(0, maxLength), attr); + // } + } + + @Override + public void replace(FilterBypass fb, int offset, int length, String text, AttributeSet attrs) throws BadLocationException { + int currentLength = fb.getDocument().getLength(); + int newLength = currentLength - length + text.length(); + + // if (newLength <= maxLength) { + super.replace(fb, offset, length, text.substring(0, maxLength), attrs); + // } + } + } + */ + ImageIcon ripIcon = new ImageIcon(mainIcon); ripButton = new JButton("Rip", ripIcon); stopButton = new JButton("Stop"); diff --git a/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java b/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java new file mode 100644 index 00000000..e247926c --- /dev/null +++ b/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java @@ -0,0 +1,30 @@ +package com.rarchives.ripme.uiUtils; + +import javax.swing.*; +import javax.swing.text.JTextComponent; +import java.awt.*; +import java.awt.datatransfer.Clipboard; +import java.awt.datatransfer.DataFlavor; +import java.awt.datatransfer.Transferable; +import java.awt.datatransfer.UnsupportedFlavorException; +import java.io.IOException; + +public class ContextActionProtections { + public static void pasteFromClipboard(JTextComponent textComponent) { + Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard(); + Transferable transferable = clipboard.getContents(new Object()); + + try { + String clipboardContent = (String) transferable.getTransferData(DataFlavor.stringFlavor); + + // Limit the pasted content to 96 characters + if (clipboardContent.length() > 96) { + clipboardContent = clipboardContent.substring(0, 96); + } + // Set the text in the JTextField + textComponent.setText(clipboardContent); + } catch (UnsupportedFlavorException | IOException unable_to_modify_text_on_paste) { + unable_to_modify_text_on_paste.printStackTrace(); + } + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java index f64ada87..940a032c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java @@ -7,6 +7,10 @@ import java.net.URL; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import com.rarchives.ripme.ripper.rippers.ChanRipper; import com.rarchives.ripme.ripper.rippers.ripperhelpers.ChanSite; @@ -27,7 +31,22 @@ public class ChanRipperTest extends RippersTest { passURLs.add(new URI("https://rbt.asia/g/thread/70643087/").toURL()); //must work with TLDs with len of 4 for (URL url : passURLs) { ChanRipper ripper = new ChanRipper(url); - ripper.setup(); + // Use CompletableFuture to run setup() asynchronously + CompletableFuture setupFuture = CompletableFuture.runAsync(() -> { + try { + ripper.setup(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + try { + // Wait for up to 5 seconds for setup() to complete + setupFuture.get(5, TimeUnit.SECONDS); + } catch (InterruptedException | ExecutionException | + TimeoutException e) { + e.printStackTrace(); // Handle exceptions as needed + } assert (ripper.canRip(url)); Assertions.assertNotNull(ripper.getWorkingDir(), "Ripper for " + url + " did not have a valid working directory."); deleteDir(ripper.getWorkingDir()); diff --git a/src/test/java/com/rarchives/ripme/ui/UIContextMenuTests.java b/src/test/java/com/rarchives/ripme/ui/UIContextMenuTests.java new file mode 100644 index 00000000..517bef4d --- /dev/null +++ b/src/test/java/com/rarchives/ripme/ui/UIContextMenuTests.java @@ -0,0 +1,187 @@ +package com.rarchives.ripme.ui; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import javax.swing.*; +import java.awt.*; +import java.awt.datatransfer.Clipboard; +import java.awt.datatransfer.StringSelection; +import java.awt.event.ActionEvent; +import java.lang.reflect.InvocationTargetException; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.jupiter.api.Assertions.fail; + +public class UIContextMenuTests { + + private JFrame frame; + private JTextField textField; + private ContextMenuMouseListener contextMenuMouseListener; + + @BeforeEach + void setUp() throws InterruptedException, InvocationTargetException { + AtomicBoolean notDone = new AtomicBoolean(true); + + SwingUtilities.invokeAndWait(() -> { + frame = new JFrame("ContextMenuMouseListener Example"); + textField = new JTextField("Hello, world!"); + + // Create an instance of ContextMenuMouseListener + contextMenuMouseListener = new ContextMenuMouseListener(textField); + + // Add ContextMenuMouseListener to JTextField + textField.addMouseListener(contextMenuMouseListener); + + frame.getContentPane().add(textField, BorderLayout.CENTER); + frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); + frame.setSize(300, 200); + frame.setVisible(true); + + notDone.set(false); + }); + + // Wait for the GUI to be fully initialized + while (notDone.get()) { + Thread.yield(); + } + } + + @AfterEach + void tearDown() { + frame.dispose(); + } + + @Test + void testCut() { + // Simulate a cut event + simulateCutEvent(); + // Add assertions if needed + } + + @Test + void testCopy() { + // Simulate a copy event + simulateCopyEvent(); + // Add assertions if needed + } + + @Test + void testPaste() { + // Simulate a paste event + simulatePasteEvent(); + // Add assertions if needed + } + + @Test + void testSelectAll() { + // Simulate a select all event + simulateSelectAllEvent(); + // Add assertions if needed + } + + @Test + void testUndo() { + // Simulate an undo event + simulateUndoEvent(); + // Add assertions if needed + } + + private void simulatePasteEvent() { + // Save the initial text content + String initialText = contextMenuMouseListener.getTextComponent().getText(); + + // Assume there is some text to paste + String textToPaste = "Text to paste"; + + // Set the text to the clipboard + Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard(); + StringSelection stringSelection = new StringSelection(textToPaste); + clipboard.setContents(stringSelection, stringSelection); + + // Simulate a paste event + contextMenuMouseListener.getTextComponent().paste(); + + // Verify that the paste operation worked + String actualText = contextMenuMouseListener.getTextComponent().getText(); + + // Check if the text was appended after the initial text + if (actualText.equals(initialText + textToPaste)) { + System.out.println("Paste operation successful. Text content matches."); + } else { + fail("Paste operation failed. Text content does not match."); + } + } + + + + + private void simulateSelectAllEvent() { + // Simulate a select all event by invoking the selectAllAction + contextMenuMouseListener.getSelectAllAction().actionPerformed(new ActionEvent(contextMenuMouseListener.getTextComponent(), ActionEvent.ACTION_PERFORMED, "")); + + // Verify that all text is selected + int expectedSelectionStart = 0; + int expectedSelectionEnd = contextMenuMouseListener.getTextComponent().getText().length(); + int actualSelectionStart = contextMenuMouseListener.getTextComponent().getSelectionStart(); + int actualSelectionEnd = contextMenuMouseListener.getTextComponent().getSelectionEnd(); + + if (expectedSelectionStart == actualSelectionStart && expectedSelectionEnd == actualSelectionEnd) { + System.out.println("Select All operation successful. Text is selected."); + } else { + fail("Select All operation failed. Text is not selected as expected."); + } + } + + private void simulateUndoEvent() { + + // Simulate an undo event by invoking the undoAction + contextMenuMouseListener.getUndoAction().actionPerformed(new ActionEvent(contextMenuMouseListener.getTextComponent(), ActionEvent.ACTION_PERFORMED, "")); + + // Verify that the undo operation worked + String expectedText = contextMenuMouseListener.getSavedString(); // Assuming the undo reverts to the saved state + String actualText = contextMenuMouseListener.getTextComponent().getText(); + + if (expectedText.equals(actualText)) { + System.out.println("Undo operation successful. Text content matches."); + } else { + fail("Undo operation failed. Text content does not match."); + } + } + + + private void simulateCopyEvent() { + // Save the initial text content + String initialText = contextMenuMouseListener.getTextComponent().getText(); + + // Simulate a copy event by invoking the copyAction + contextMenuMouseListener.getCopyAction().actionPerformed(new ActionEvent(contextMenuMouseListener.getTextComponent(), ActionEvent.ACTION_PERFORMED, "")); + + // Verify that the copy operation worked + String actualText = contextMenuMouseListener.getDebugSavedString(); + + if (initialText.equals(actualText)) { + System.out.println("Copy operation successful. Text content matches."); + } else { + fail("Copy operation failed. Text content does not match."); + } + } + + private void simulateCutEvent() { + // Save the initial text content + String initialText = contextMenuMouseListener.getTextComponent().getText(); + + // Simulate a cut event by invoking the cutAction + contextMenuMouseListener.getCutAction().actionPerformed(new ActionEvent(contextMenuMouseListener.getTextComponent(), ActionEvent.ACTION_PERFORMED, "")); + + // Verify that the cut operation worked + String actualText = contextMenuMouseListener.getDebugSavedString(); + + if (initialText.equals(actualText)) { + System.out.println("Cut operation successful. Text content matches."); + } else { + fail("Cut operation failed. Text content does not match."); + } + } +} From 9e6bb0c8c4a5507043c966f4460dacdf423ed41b Mon Sep 17 00:00:00 2001 From: Undid-Iridium <24619207+Undid-Iridium@users.noreply.github.com> Date: Sat, 9 Dec 2023 04:48:33 -0500 Subject: [PATCH 436/512] Refactor DanbooruRipper to use OkHttpClient Replaced the existing request method in the DanbooruRipper class with OkHttpClient for better performance. Implemented error handling and response checking to ensure the http connection's success. The request headers were also updated to optimize and secure the connection. In addition, connection was able to work via Connection connection = Jsoup.connect("https://danbooru.donmai.us/posts?page=1&tags=rakeemspoon") .ignoreContentType(true) .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0") .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8") .header("Accept-Language", "en-US,en;q=0.5") .header("Upgrade-Insecure-Requests", "1") .header("Sec-Fetch-Dest", "document") .header("Sec-Fetch-Mode", "navigate") .header("Sec-Fetch-Site", "same-origin") .header("Sec-Fetch-User", "?1") .header("Pragma", "no-cache") .header("Cache-Control", "no-cache") .header("Referer", "https://danbooru.donmai.us/posts?page=4&tags=rakeemspoon") .header("Accept-Encoding", "gzip, deflate, br"); However, the returned data was gibberish - I had no functional idea on how to fix it. --- .../ripme/ripper/rippers/DanbooruRipper.java | 113 ++++++++++++++++-- 1 file changed, 101 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java index fb0bb233..26e4f1f5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java @@ -3,18 +3,29 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractJSONRipper; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; import org.json.JSONArray; import org.json.JSONObject; +import org.jsoup.Connection; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStreamReader; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.zip.GZIPInputStream; public class DanbooruRipper extends AbstractJSONRipper { private static final String DOMAIN = "danbooru.donmai.us", @@ -42,29 +53,107 @@ public class DanbooruRipper extends AbstractJSONRipper { return "https://" + getDomain() + "/posts.json?page=" + num + "&tags=" + getTag(url); } - @Override - protected JSONObject getFirstPage() throws IOException { - String newCompatibleJSON = "{ resources:" + Http.url(getPage(1)).getJSONArray() + " }"; + private static final Map SESSION_COOKIE; + static { + SESSION_COOKIE = new TreeMap(); + SESSION_COOKIE.put("_danbooru2_session", "9V8N6tN5EW9gMFt%2BCrX4urKQD7VDwyLDcTqaTouqs%2FaOCasOJvCEWxNpm87RrDeK7Q51BVHjaS%2BDJQxDbmQNY%2BftVereWltgkFCOqcIweYRdKIIRwBSBJuFzhwz8raelfHZeDT9XHjUqZ6ShnWc0HVmB6FImIhxKqvU9c0pywoiY%2Fv6tSOmb9kCXLGVbP6ltOJOUR7fUyXNSz56YOZ7ycCtbTrOLK1abSuUFg1nLREh2pDqbZnHskEvYPdAmfejsgpmqnABzJH%2B1mt8j53y0%2BIC0F%2BE1n8ho1o77pKsOJuLiHTspxZho2PJ3JM%2Fa1eeA0ydlgJ5DKeHly0VwRZeNPDOPg%2F9c2VTEoaqSSnAyYWuAtilkMO52VGqcytqSlGtf6tlCMg%3D%3D--m98PPTXsgxn8A0dm--cPRKozLSLkwE4sJvirVU1g%3D%3D"); + } - return new JSONObject(newCompatibleJSON); + private final String userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0"; + @Override + protected JSONObject getFirstPage() throws MalformedURLException { + + OkHttpClient client = new OkHttpClient.Builder() + .readTimeout(60, TimeUnit.SECONDS) + .writeTimeout(60, TimeUnit.SECONDS) + .build(); + + Request request = new Request.Builder() + .url(getPage(1)) // make sure to implement getPage method + .header("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1") + .header("Accept", "application/json,text/javascript,*/*;q=0.01") + .header("Accept-Language", "en-US,en;q=0.9") + .header("Sec-Fetch-Dest", "empty") + .header("Sec-Fetch-Mode", "cors") + .header("Sec-Fetch-Site", "same-origin") + .header("Referer", "https://danbooru.donmai.us/") + .header("X-Requested-With", "XMLHttpRequest") + .header("Connection", "keep-alive") + .build(); + + Response response = null; + try { + response = client.newCall(request).execute(); + if (!response.isSuccessful()) throw new IOException("Unexpected code " + response); + + // Response body is automatically decompressed + String responseData = response.body().string(); + // Parsing the responseData to a JSONArray + JSONArray jsonArray = new JSONArray(responseData); + System.out.println(jsonArray.toString()); + + String newCompatibleJSON = "{ \"resources\":" + jsonArray.toString() + " }"; + return new JSONObject(newCompatibleJSON); + + } catch (IOException e) { + e.printStackTrace(); + } finally { + if(response !=null) { + response.body().close(); + } + } + return null; // Return null or a default value in case of error } @Override protected JSONObject getNextPage(JSONObject doc) throws IOException { currentPageNum++; - JSONArray resourcesJSONArray = Http.url(getPage(currentPageNum)).getJSONArray(); + OkHttpClient client = new OkHttpClient.Builder() + .readTimeout(60, TimeUnit.SECONDS) + .writeTimeout(60, TimeUnit.SECONDS) + .build(); - int resourcesJSONArrayLength = resourcesJSONArray.length(); + Request request = new Request.Builder() + .url(getPage(currentPageNum)) // make sure to implement getPage method + .header("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1") + .header("Accept", "application/json,text/javascript,*/*;q=0.01") + .header("Accept-Language", "en-US,en;q=0.9") + .header("Sec-Fetch-Dest", "empty") + .header("Sec-Fetch-Mode", "cors") + .header("Sec-Fetch-Site", "same-origin") + .header("Referer", "https://danbooru.donmai.us/") + .header("X-Requested-With", "XMLHttpRequest") + .header("Connection", "keep-alive") + .build(); - if (resourcesJSONArrayLength == 0) { - currentPageNum = 0; - throw new IOException("No more images in the next page"); + Response response = null; + try { + response = client.newCall(request).execute(); + if (!response.isSuccessful()) throw new IOException("Unexpected code " + response); + + // Response body is automatically decompressed + String responseData = response.body().string(); + // Parsing the responseData to a JSONArray + + JSONArray jsonArray = new JSONArray(responseData); + if(!jsonArray.isEmpty()){ + System.out.println(jsonArray); + + String newCompatibleJSON = "{ \"resources\":" + jsonArray + " }"; + return new JSONObject(newCompatibleJSON); + } + + + } catch (IOException e) { + e.printStackTrace(); + } finally { + if(response !=null) { + response.body().close(); + } } + return null; // Return null or a default value in case of error - String newCompatibleJSON = "{ resources:" + resourcesJSONArray + " }"; - - return new JSONObject(newCompatibleJSON); } @Override From 9c18f452cde046b4dadf6f30026bc282785f647c Mon Sep 17 00:00:00 2001 From: Undid-Iridium <24619207+Undid-Iridium@users.noreply.github.com> Date: Sat, 9 Dec 2023 04:54:06 -0500 Subject: [PATCH 437/512] Add OkHttp dependency in build.gradle.kts A new implementation line was added to the build.gradle.kts file for OkHttp library. This change is needed in order to extend the project's capabilities for HTTP network requests for ones with encryption/compression. --- build.gradle.kts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index e9b1ce55..3266fae6 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -31,6 +31,7 @@ dependencies { implementation("org.apache.httpcomponents:httpmime:4.5.14") implementation("org.apache.logging.log4j:log4j-api:2.20.0") implementation("org.apache.logging.log4j:log4j-core:2.20.0") + implementation("com.squareup.okhttp3:okhttp:4.12.0") implementation("org.graalvm.js:js:22.3.2") testImplementation(enforcedPlatform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") @@ -68,7 +69,7 @@ tasks.withType { attributes["Implementation-Version"] = archiveVersion attributes["Multi-Release"] = "true" } - + // To add all of the dependencies otherwise a "NoClassDefFoundError" error from(sourceSets.main.get().output) From 4f10a8000b73a6a18217ff8e049128c68aaf1210 Mon Sep 17 00:00:00 2001 From: Undid-Iridium <24619207+Undid-Iridium@users.noreply.github.com> Date: Sat, 9 Dec 2023 04:56:44 -0500 Subject: [PATCH 438/512] Add OkHttpClient to DanbooruRipper constructor - Some of these headers may not be needed but.. oh well. --- .../ripme/ripper/rippers/DanbooruRipper.java | 27 +++++-------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java index 26e4f1f5..120f1660 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java @@ -30,6 +30,7 @@ import java.util.zip.GZIPInputStream; public class DanbooruRipper extends AbstractJSONRipper { private static final String DOMAIN = "danbooru.donmai.us", HOST = "danbooru"; + private final OkHttpClient client; private Pattern gidPattern = null; @@ -37,6 +38,10 @@ public class DanbooruRipper extends AbstractJSONRipper { public DanbooruRipper(URL url) throws IOException { super(url); + this.client = new OkHttpClient.Builder() + .readTimeout(60, TimeUnit.SECONDS) + .writeTimeout(60, TimeUnit.SECONDS) + .build(); } @Override @@ -53,12 +58,6 @@ public class DanbooruRipper extends AbstractJSONRipper { return "https://" + getDomain() + "/posts.json?page=" + num + "&tags=" + getTag(url); } - private static final Map SESSION_COOKIE; - static { - SESSION_COOKIE = new TreeMap(); - SESSION_COOKIE.put("_danbooru2_session", "9V8N6tN5EW9gMFt%2BCrX4urKQD7VDwyLDcTqaTouqs%2FaOCasOJvCEWxNpm87RrDeK7Q51BVHjaS%2BDJQxDbmQNY%2BftVereWltgkFCOqcIweYRdKIIRwBSBJuFzhwz8raelfHZeDT9XHjUqZ6ShnWc0HVmB6FImIhxKqvU9c0pywoiY%2Fv6tSOmb9kCXLGVbP6ltOJOUR7fUyXNSz56YOZ7ycCtbTrOLK1abSuUFg1nLREh2pDqbZnHskEvYPdAmfejsgpmqnABzJH%2B1mt8j53y0%2BIC0F%2BE1n8ho1o77pKsOJuLiHTspxZho2PJ3JM%2Fa1eeA0ydlgJ5DKeHly0VwRZeNPDOPg%2F9c2VTEoaqSSnAyYWuAtilkMO52VGqcytqSlGtf6tlCMg%3D%3D--m98PPTXsgxn8A0dm--cPRKozLSLkwE4sJvirVU1g%3D%3D"); - } - private final String userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0"; @Override protected JSONObject getFirstPage() throws MalformedURLException { @@ -109,10 +108,7 @@ public class DanbooruRipper extends AbstractJSONRipper { protected JSONObject getNextPage(JSONObject doc) throws IOException { currentPageNum++; - OkHttpClient client = new OkHttpClient.Builder() - .readTimeout(60, TimeUnit.SECONDS) - .writeTimeout(60, TimeUnit.SECONDS) - .build(); + Request request = new Request.Builder() .url(getPage(currentPageNum)) // make sure to implement getPage method @@ -126,25 +122,17 @@ public class DanbooruRipper extends AbstractJSONRipper { .header("X-Requested-With", "XMLHttpRequest") .header("Connection", "keep-alive") .build(); - Response response = null; try { response = client.newCall(request).execute(); if (!response.isSuccessful()) throw new IOException("Unexpected code " + response); - // Response body is automatically decompressed String responseData = response.body().string(); - // Parsing the responseData to a JSONArray - JSONArray jsonArray = new JSONArray(responseData); if(!jsonArray.isEmpty()){ - System.out.println(jsonArray); - String newCompatibleJSON = "{ \"resources\":" + jsonArray + " }"; return new JSONObject(newCompatibleJSON); } - - } catch (IOException e) { e.printStackTrace(); } finally { @@ -152,8 +140,7 @@ public class DanbooruRipper extends AbstractJSONRipper { response.body().close(); } } - return null; // Return null or a default value in case of error - + return null; } @Override From 10538c39f7855ccadd29620697f0c4d4a851ae53 Mon Sep 17 00:00:00 2001 From: Undid-Iridium <24619207+Undid-Iridium@users.noreply.github.com> Date: Sat, 9 Dec 2023 05:42:21 -0500 Subject: [PATCH 439/512] Update regex pattern in getTag method The regular expression pattern used to match tags in URLs in the DanbooruRipper class's getTag method has been updated. Handles both /posts?tags=rakeemspoon&z=2 and /posts?tags=rakeemspoon ([^&]*) -> anything but & (?:&z=([0-9]+))? -> optional regex -> if & exists, then we check z=numbers --- .../java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java index 120f1660..ea1365b9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java @@ -172,7 +172,7 @@ public class DanbooruRipper extends AbstractJSONRipper { } private String getTag(URL url) throws MalformedURLException { - gidPattern = Pattern.compile("https?://danbooru.donmai.us/(posts)?.*([?&]tags=([a-zA-Z0-9$_.+!*'(),%-]+))(&|(#.*)?$)"); + gidPattern = Pattern.compile("https?://danbooru.donmai.us/(posts)?.*([?&]tags=([^&]*)(?:&z=([0-9]+))?$)"); Matcher m = gidPattern.matcher(url.toExternalForm()); if (m.matches()) { From 10a1b1218066ed6e73fd454025338d64d20951df Mon Sep 17 00:00:00 2001 From: Undid-Iridium <24619207+Undid-Iridium@users.noreply.github.com> Date: Sat, 9 Dec 2023 05:55:01 -0500 Subject: [PATCH 440/512] Refactor DanbooruRipper's page fetching logic The logic for fetching current and next pages from DanbooruRipper has been streamlined and simplified - they basically can run off the same logic. --- .../ripme/ripper/rippers/DanbooruRipper.java | 53 +++---------------- 1 file changed, 8 insertions(+), 45 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java index ea1365b9..9496bb57 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java @@ -6,6 +6,7 @@ import com.rarchives.ripme.utils.Utils; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; +import org.jetbrains.annotations.Nullable; import org.json.JSONArray; import org.json.JSONObject; import org.jsoup.Connection; @@ -61,57 +62,18 @@ public class DanbooruRipper extends AbstractJSONRipper { private final String userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0"; @Override protected JSONObject getFirstPage() throws MalformedURLException { - - OkHttpClient client = new OkHttpClient.Builder() - .readTimeout(60, TimeUnit.SECONDS) - .writeTimeout(60, TimeUnit.SECONDS) - .build(); - - Request request = new Request.Builder() - .url(getPage(1)) // make sure to implement getPage method - .header("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1") - .header("Accept", "application/json,text/javascript,*/*;q=0.01") - .header("Accept-Language", "en-US,en;q=0.9") - .header("Sec-Fetch-Dest", "empty") - .header("Sec-Fetch-Mode", "cors") - .header("Sec-Fetch-Site", "same-origin") - .header("Referer", "https://danbooru.donmai.us/") - .header("X-Requested-With", "XMLHttpRequest") - .header("Connection", "keep-alive") - .build(); - - Response response = null; - try { - response = client.newCall(request).execute(); - if (!response.isSuccessful()) throw new IOException("Unexpected code " + response); - - // Response body is automatically decompressed - String responseData = response.body().string(); - // Parsing the responseData to a JSONArray - JSONArray jsonArray = new JSONArray(responseData); - System.out.println(jsonArray.toString()); - - String newCompatibleJSON = "{ \"resources\":" + jsonArray.toString() + " }"; - return new JSONObject(newCompatibleJSON); - - } catch (IOException e) { - e.printStackTrace(); - } finally { - if(response !=null) { - response.body().close(); - } - } - return null; // Return null or a default value in case of error + return getCurrentPage(); } @Override protected JSONObject getNextPage(JSONObject doc) throws IOException { - currentPageNum++; - - + return getCurrentPage(); + } + @Nullable + private JSONObject getCurrentPage() throws MalformedURLException { Request request = new Request.Builder() - .url(getPage(currentPageNum)) // make sure to implement getPage method + .url(getPage(currentPageNum)) .header("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1") .header("Accept", "application/json,text/javascript,*/*;q=0.01") .header("Accept-Language", "en-US,en;q=0.9") @@ -123,6 +85,7 @@ public class DanbooruRipper extends AbstractJSONRipper { .header("Connection", "keep-alive") .build(); Response response = null; + currentPageNum++; try { response = client.newCall(request).execute(); if (!response.isSuccessful()) throw new IOException("Unexpected code " + response); From ba29da02b3847e80cd70327b7ccc5edd5b8e9c0b Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 9 Dec 2023 15:00:08 +0100 Subject: [PATCH 441/512] flaky tests chan, wordpresscomic --- .../com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java | 2 ++ .../ripme/tst/ripper/rippers/WordpressComicRipperTest.java | 1 + 2 files changed, 3 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java index 940a032c..016ba218 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java @@ -17,10 +17,12 @@ import com.rarchives.ripme.ripper.rippers.ripperhelpers.ChanSite; import com.rarchives.ripme.utils.Http; import org.jsoup.nodes.Document; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class ChanRipperTest extends RippersTest { @Test + @Tag("flaky") public void testChanURLPasses() throws IOException, URISyntaxException { List passURLs = new ArrayList<>(); // URLs that should work diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java index b27234f9..d0649aa9 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java @@ -102,6 +102,7 @@ public class WordpressComicRipperTest extends RippersTest { Assertions.assertEquals("8muses.download_lustomic-playkittens-josh-samuel-porn-comics-8-muses", ripper.getAlbumTitle(url)); } @Test + @Tag("flaky") public void test_spyingwithlana_download() throws IOException, URISyntaxException { WordpressComicRipper ripper = new WordpressComicRipper( new URI("http://spyingwithlana.com/comic/the-big-hookup/").toURL()); From d40eed3fda37b8ac175e20f55224c545d2d7ad15 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 9 Dec 2023 15:06:15 +0100 Subject: [PATCH 442/512] switch off UIContextMenuTests, headless on server makes them fail. --- src/test/java/com/rarchives/ripme/ui/UIContextMenuTests.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/ui/UIContextMenuTests.java b/src/test/java/com/rarchives/ripme/ui/UIContextMenuTests.java index 517bef4d..32dcdd9b 100644 --- a/src/test/java/com/rarchives/ripme/ui/UIContextMenuTests.java +++ b/src/test/java/com/rarchives/ripme/ui/UIContextMenuTests.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ui; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import javax.swing.*; @@ -14,6 +15,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.jupiter.api.Assertions.fail; +// these tests do not run on a server, as it is headless +@Tag("flaky") public class UIContextMenuTests { private JFrame frame; From 867e933597fd3566ad700b223f62683621e2cc87 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 9 Dec 2023 15:10:10 +0100 Subject: [PATCH 443/512] yuvutu test flaky --- .../rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java index d8c8d756..cc84c8d5 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YuvutuRipperTest.java @@ -7,10 +7,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.YuvutuRipper; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class YuvutuRipperTest extends RippersTest { @Test + @Tag("flaky") public void testYuvutuAlbum1() throws IOException, URISyntaxException { YuvutuRipper ripper = new YuvutuRipper(new URI("http://www.yuvutu.com/modules.php?name=YuGallery&action=view&set_id=127013").toURL()); testRipper(ripper); From 879c322e7be69513e71c589df0a13d7007b3cb9e Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 9 Dec 2023 15:25:36 +0100 Subject: [PATCH 444/512] new URI instead of new URL, imgabam, imagevenue --- .../com/rarchives/ripme/ripper/rippers/ImagebamRipper.java | 6 ++++-- .../rarchives/ripme/ripper/rippers/ImagevenueRipper.java | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index 35e8d0b5..9109466d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -6,6 +6,8 @@ import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -148,8 +150,8 @@ public class ImagebamRipper extends AbstractHTMLRipper { prefix = String.format("%03d_", index); } - addURLToDownload(new URL(imgsrc), prefix); - } catch (IOException e) { + addURLToDownload(new URI(imgsrc).toURL(), prefix); + } catch (IOException | URISyntaxException e) { LOGGER.error("[!] Exception while loading/parsing " + this.url, e); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java index cc58c561..4691c7c6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -108,8 +110,8 @@ public class ImagevenueRipper extends AbstractHTMLRipper { if (Utils.getConfigBoolean("download.save_order", true)) { prefix = String.format("%03d_", index); } - addURLToDownload(new URL(imgsrc), prefix); - } catch (IOException e) { + addURLToDownload(new URI(imgsrc).toURL(), prefix); + } catch (IOException | URISyntaxException e) { LOGGER.error("[!] Exception while loading/parsing " + this.url, e); } } From 8f279a68ecbf2f26a4e6f1c2fc2f5418de8c5990 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 9 Dec 2023 15:29:56 +0100 Subject: [PATCH 445/512] new URI instead of new URL, mangadex, vk. --- .../ripme/ripper/AbstractJSONRipper.java | 4 ++-- .../ripme/ripper/rippers/MangadexRipper.java | 12 +++++++----- .../rarchives/ripme/ripper/rippers/VkRipper.java | 6 ++++-- .../tst/ripper/rippers/MangadexRipperTest.java | 16 ++++++++++------ 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 538a4b5d..9dd891c8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -37,7 +37,7 @@ public abstract class AbstractJSONRipper extends AbstractRipper { @Override public abstract String getHost(); - protected abstract JSONObject getFirstPage() throws IOException; + protected abstract JSONObject getFirstPage() throws IOException, URISyntaxException; protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException { throw new IOException("getNextPage not implemented"); } @@ -62,7 +62,7 @@ public abstract class AbstractJSONRipper extends AbstractRipper { } @Override - public void rip() throws IOException { + public void rip() throws IOException, URISyntaxException { int index = 0; LOGGER.info("Retrieving " + this.url); sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm()); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java index 1e845d61..8c6c9227 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MangadexRipper.java @@ -8,6 +8,8 @@ import org.json.JSONObject; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; @@ -79,14 +81,14 @@ public class MangadexRipper extends AbstractJSONRipper { @Override - public JSONObject getFirstPage() throws IOException { + public JSONObject getFirstPage() throws IOException, URISyntaxException { // Get the chapter ID String chapterID = getChapterID(url.toExternalForm()); String mangaID = getMangaID(url.toExternalForm()); if (mangaID != null) { - return Http.url(new URL(mangaApiEndPoint + mangaID)).getJSON(); + return Http.url(new URI(mangaApiEndPoint + mangaID).toURL()).getJSON(); } else - return Http.url(new URL(chapterApiEndPoint + chapterID)).getJSON(); + return Http.url(new URI(chapterApiEndPoint + chapterID).toURL()).getJSON(); } @Override @@ -129,8 +131,8 @@ public class MangadexRipper extends AbstractJSONRipper { for (Double aDouble : treeMap.keySet()) { double key = (double) aDouble; try { - chapterJSON = Http.url(new URL(chapterApiEndPoint + treeMap.get(key))).getJSON(); - } catch (IOException e) { + chapterJSON = Http.url(new URI(chapterApiEndPoint + treeMap.get(key)).toURL()).getJSON(); + } catch (IOException | URISyntaxException e) { e.printStackTrace(); } sendUpdate(RipStatusMessage.STATUS.LOADING_RESOURCE, "chapter " + key); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java index b364a5ae..c6394bb8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.*; import java.util.regex.Matcher; @@ -137,13 +139,13 @@ public class VkRipper extends AbstractJSONRipper { } @Override - public void rip() throws IOException { + public void rip() throws IOException, URISyntaxException { if (this.url.toExternalForm().contains("/videos")) { RIP_TYPE = RipType.VIDEO; JSONObject json = getFirstPage(); List URLs = getURLsFromJSON(json); for (int index = 0; index < URLs.size(); index ++) { - downloadURL(new URL(URLs.get(index)), index); + downloadURL(new URI(URLs.get(index)).toURL(), index); } waitForThreads(); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MangadexRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MangadexRipperTest.java index fe957e32..3bcec8c8 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MangadexRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MangadexRipperTest.java @@ -2,22 +2,26 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.MangadexRipper; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -public class MangadexRipperTest extends RippersTest{ +public class MangadexRipperTest extends RippersTest { + @Test + @Tag("flaky") public void testRip() throws IOException, URISyntaxException { MangadexRipper ripper = new MangadexRipper(new URI("https://mangadex.org/chapter/467904/").toURL()); testRipper(ripper); } - public class testMangaRip extends RippersTest{ - public void testRip() throws IOException, URISyntaxException { - MangadexRipper ripper = new MangadexRipper(new URI("https://mangadex.org/title/44625/this-croc-will-die-in-100-days").toURL()); - testRipper(ripper); - } + @Test + @Tag("flaky") + public void test2() throws IOException, URISyntaxException { + MangadexRipper ripper = new MangadexRipper(new URI("https://mangadex.org/title/44625/this-croc-will-die-in-100-days").toURL()); + testRipper(ripper); } } From 8b9e4b98de9a1f6522a3c6afcfb4fbfb66626e80 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 9 Dec 2023 16:03:57 +0100 Subject: [PATCH 446/512] new URI instead of new URL, motherless, nfsfw. --- .../ripme/ripper/rippers/MotherlessRipper.java | 10 ++++++---- .../rarchives/ripme/ripper/rippers/NfsfwRipper.java | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java index 46331c7b..ee657fb9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -66,13 +68,13 @@ public class MotherlessRipper extends AbstractHTMLRipper { } @Override - public Document getNextPage(Document doc) throws IOException { + public Document getNextPage(Document doc) throws IOException, URISyntaxException { Elements nextPageLink = doc.head().select("link[rel=next]"); if (nextPageLink.isEmpty()) { throw new IOException("Last page reached"); } else { String referrerLink = doc.head().select("link[rel=canonical]").first().attr("href"); - URL nextURL = new URL(this.url, nextPageLink.first().attr("href")); + URL nextURL = this.url.toURI().resolve(nextPageLink.first().attr("href")).toURL(); return Http.url(nextURL).referrer(referrerLink).get(); } } @@ -180,11 +182,11 @@ public class MotherlessRipper extends AbstractHTMLRipper { if (Utils.getConfigBoolean("download.save_order", true)) { prefix = String.format("%03d_", index); } - addURLToDownload(new URL(file), prefix); + addURLToDownload(new URI(file).toURL(), prefix); } else { LOGGER.warn("[!] could not find '__fileurl' at " + url); } - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { LOGGER.error("[!] Exception while loading/parsing " + this.url, e); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java index 03a4717f..595dd452 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -104,13 +106,13 @@ public class NfsfwRipper extends AbstractHTMLRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { // always start on the first page of an album // (strip the options after the '?') String u = url.toExternalForm(); if (u.contains("?")) { u = u.substring(0, u.indexOf("?")); - return new URL(u); + return new URI(u).toURL(); } else { return url; } @@ -220,8 +222,8 @@ public class NfsfwRipper extends AbstractHTMLRipper { if (file.startsWith("/")) { file = "http://nfsfw.com" + file; } - addURLToDownload(new URL(file), getPrefix(index), this.subdir); - } catch (IOException e) { + addURLToDownload(new URI(file).toURL(), getPrefix(index), this.subdir); + } catch (IOException | URISyntaxException e) { LOGGER.error("[!] Exception while loading/parsing " + this.url, e); } } From 43ebb8d643b0eb41da1ea3461c964c6bbaeeef2b Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 9 Dec 2023 16:13:03 +0100 Subject: [PATCH 447/512] new URI instead of new URL, pornhub, sta. --- .../ripme/ripper/rippers/PornhubRipper.java | 14 ++++++++------ .../rarchives/ripme/ripper/rippers/StaRipper.java | 10 ++++++---- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java index a215102e..a2ce4a19 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Path; import java.util.ArrayList; @@ -47,12 +49,12 @@ public class PornhubRipper extends AbstractHTMLRipper { } @Override - public Document getNextPage(Document page) throws IOException { + public Document getNextPage(Document page) throws IOException, URISyntaxException { Elements nextPageLink = page.select("li.page_next > a"); if (nextPageLink.isEmpty()){ throw new IOException("No more pages"); } else { - URL nextURL = new URL(this.url, nextPageLink.first().attr("href")); + URL nextURL = this.url.toURI().resolve(nextPageLink.first().attr("href")).toURL(); return Http.url(nextURL).get(); } } @@ -83,13 +85,13 @@ public class PornhubRipper extends AbstractHTMLRipper { } } - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { // always start on the first page of an album // (strip the options after the '?') String u = url.toExternalForm(); if (u.contains("?")) { u = u.substring(0, u.indexOf("?")); - return new URL(u); + return new URI(u).toURL(); } else { return url; } @@ -159,10 +161,10 @@ public class PornhubRipper extends AbstractHTMLRipper { prefix = String.format("%03d_", index); } - URL imgurl = new URL(url, imgsrc); + URL imgurl = url.toURI().resolve(imgsrc).toURL(); addURLToDownload(imgurl, prefix); - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { LOGGER.error("[!] Exception while loading/parsing " + this.url, e); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java index ad73e452..ac7414dd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; @@ -55,10 +57,10 @@ public class StaRipper extends AbstractHTMLRipper { Document thumbPage = null; if (checkURL(thumbPageURL)) { try { - Connection.Response resp = Http.url(new URL(thumbPageURL)).response(); + Connection.Response resp = Http.url(new URI(thumbPageURL).toURL()).response(); cookies.putAll(resp.cookies()); thumbPage = resp.parse(); - } catch (MalformedURLException e) { + } catch (MalformedURLException | URISyntaxException e) { LOGGER.info(thumbPageURL + " is a malformed URL"); } catch (IOException e) { LOGGER.info(e.getMessage()); @@ -75,9 +77,9 @@ public class StaRipper extends AbstractHTMLRipper { private boolean checkURL(String url) { try { - new URL(url); + new URI(url).toURL(); return true; - } catch (MalformedURLException e) { + } catch (MalformedURLException | URISyntaxException e) { return false; } } From 81a77f09f81fa5f2241cfc41dccf45e381e2b7ce Mon Sep 17 00:00:00 2001 From: joroto <47276635+joroto@users.noreply.github.com> Date: Wed, 13 Dec 2023 13:44:39 +0200 Subject: [PATCH 448/512] JPG3 (JPG Fish) ripper added --- .../ripme/ripper/rippers/Jpg3Ripper.java | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java new file mode 100644 index 00000000..e6a7344a --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java @@ -0,0 +1,61 @@ +package com.rarchives.ripme.ripper.rippers; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +public class Jpg3Ripper extends AbstractHTMLRipper { + + public Jpg3Ripper(URL url) throws IOException { + super(url); + } + + @Override + public String getDomain() { + return "jpg3.su"; + } + + @Override + public String getHost() { + return "jpg3"; + } + + @Override + public List getURLsFromPage(Document page) { + List urls = new ArrayList<>(); + + for (Element el : page.select(".image-container > img")) { + urls.add(el.attr("src").replaceAll("\\.md", "")); + } + + return urls; + } + + @Override + public Document getNextPage(Document page) throws IOException, URISyntaxException { + String href = page.select("[data-pagination='next']").attr("href"); + if (!href.isEmpty()) { + return Http.url(href).get(); + } else { + return null; + } + } + + @Override + public String getGID(URL url) throws MalformedURLException { + return url.toString().split("/")[url.toString().split("/").length - 1]; + } + + @Override + protected void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), null); + } +} From e185b2acf32576a5161eca387e392efb2bb281be Mon Sep 17 00:00:00 2001 From: georgi marinov Date: Wed, 13 Dec 2023 14:06:24 +0200 Subject: [PATCH 449/512] sanitizeURL added to jpg3 ripper to handle when link to other than the first page of the album is inputted --- .../com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java index e6a7344a..bdc9af26 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java @@ -7,6 +7,7 @@ import org.jsoup.nodes.Element; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -39,6 +40,14 @@ public class Jpg3Ripper extends AbstractHTMLRipper { return urls; } + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + String u = url.toExternalForm(); + u = u.replaceAll("https?://jpg3.su/a/([^/]+)/?.*", "https://jpg3.su/a/$1"); + LOGGER.debug("Changed URL from " + url + " to " + u); + return new URL(u); + } + @Override public Document getNextPage(Document page) throws IOException, URISyntaxException { String href = page.select("[data-pagination='next']").attr("href"); From eddcd12e61c4d6735e62bd4448ef78d4226af953 Mon Sep 17 00:00:00 2001 From: georgi marinov Date: Wed, 13 Dec 2023 14:09:52 +0200 Subject: [PATCH 450/512] remove not used import --- src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java index bdc9af26..6a8cd5b5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java @@ -7,7 +7,6 @@ import org.jsoup.nodes.Element; import java.io.IOException; import java.net.MalformedURLException; -import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; From f5153de8cf157284040ea91990fdf25053535434 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 29 Dec 2023 11:08:08 +0100 Subject: [PATCH 451/512] new URI instead of new URL, erotiv. --- .../com/rarchives/ripme/ripper/rippers/ErotivRipper.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java index a8cd58c2..04511085 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -56,8 +58,8 @@ public class ErotivRipper extends AbstractHTMLRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return new URL(url.toExternalForm().replaceAll("https?://www.erotiv.io", "https://erotiv.io")); + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { + return new URI(url.toExternalForm().replaceAll("https?://www.erotiv.io", "https://erotiv.io")).toURL(); } @Override From c3a4df47e9b9a236a590606ed160a1fa4bd9ab59 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 29 Dec 2023 11:32:04 +0100 Subject: [PATCH 452/512] new URI instead of new URL, rule34, getFirstPage. --- .../com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 6 +++--- .../com/rarchives/ripme/ripper/AbstractJSONRipper.java | 2 +- .../java/com/rarchives/ripme/ripper/AbstractRipper.java | 6 +++--- src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java | 2 +- .../java/com/rarchives/ripme/ripper/RipperInterface.java | 2 +- .../rarchives/ripme/ripper/rippers/AerisdiesRipper.java | 3 ++- .../com/rarchives/ripme/ripper/rippers/BatoRipper.java | 3 ++- .../com/rarchives/ripme/ripper/rippers/ChanRipper.java | 3 ++- .../rarchives/ripme/ripper/rippers/CheveretoRipper.java | 3 ++- .../com/rarchives/ripme/ripper/rippers/EHentaiRipper.java | 2 +- .../rarchives/ripme/ripper/rippers/EightmusesRipper.java | 2 +- .../rarchives/ripme/ripper/rippers/EroShareRipper.java | 2 +- .../com/rarchives/ripme/ripper/rippers/EromeRipper.java | 3 ++- .../com/rarchives/ripme/ripper/rippers/FlickrRipper.java | 2 +- .../ripme/ripper/rippers/GirlsOfDesireRipper.java | 3 ++- .../com/rarchives/ripme/ripper/rippers/HbrowseRipper.java | 3 ++- .../rarchives/ripme/ripper/rippers/Hentai2readRipper.java | 3 ++- .../rarchives/ripme/ripper/rippers/HentaifoxRipper.java | 3 ++- .../com/rarchives/ripme/ripper/rippers/HitomiRipper.java | 3 ++- .../rarchives/ripme/ripper/rippers/HqpornerRipper.java | 3 ++- .../rarchives/ripme/ripper/rippers/ImagebamRipper.java | 2 +- .../rarchives/ripme/ripper/rippers/ImagefapRipper.java | 2 +- .../ripme/ripper/rippers/MyhentaicomicsRipper.java | 3 ++- .../com/rarchives/ripme/ripper/rippers/NfsfwRipper.java | 2 +- .../com/rarchives/ripme/ripper/rippers/Rule34Ripper.java | 8 +++++--- .../rarchives/ripme/ripper/rippers/SoundgasmRipper.java | 2 +- .../rarchives/ripme/ripper/rippers/ViewcomicRipper.java | 3 ++- .../rarchives/ripme/ripper/rippers/WebtoonsRipper.java | 3 ++- .../ripme/ripper/rippers/WordpressComicRipper.java | 3 ++- .../rarchives/ripme/ripper/rippers/XhamsterRipper.java | 5 +++-- .../com/rarchives/ripme/ripper/rippers/XvideosRipper.java | 3 ++- .../com/rarchives/ripme/ripper/rippers/ZizkiRipper.java | 3 ++- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 2 +- .../ripme/tst/ripper/rippers/ChanRipperTest.java | 4 ++-- 34 files changed, 62 insertions(+), 42 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index c906f9da..8c26c903 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -41,11 +41,11 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { protected abstract String getDomain(); public abstract String getHost(); - protected Document getFirstPage() throws IOException { + protected Document getFirstPage() throws IOException, URISyntaxException { return Http.url(url).get(); } - protected Document getCachedFirstPage() throws IOException { + protected Document getCachedFirstPage() throws IOException, URISyntaxException { if (cachedFirstPage == null) { cachedFirstPage = getFirstPage(); } @@ -462,7 +462,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { * URL to define how the working directory should be saved. */ @Override - public void setWorkingDir(URL url) throws IOException { + public void setWorkingDir(URL url) throws IOException, URISyntaxException { Path wd = Utils.getWorkingDirectory(); // TODO - change to nio String path = wd.toAbsolutePath().toString(); diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 9dd891c8..1d8e688a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -283,7 +283,7 @@ public abstract class AbstractJSONRipper extends AbstractRipper { * IOException */ @Override - public void setWorkingDir(URL url) throws IOException { + public void setWorkingDir(URL url) throws IOException, URISyntaxException { Path wd = Utils.getWorkingDirectory(); String title; if (Utils.getConfigBoolean("album_titles.save", true)) { diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index fa58b5c1..481c694f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -188,7 +188,7 @@ public abstract class AbstractRipper * @throws IOException * Always be prepared. */ - public void setup() throws IOException { + public void setup() throws IOException, URISyntaxException { setWorkingDir(this.url); // we do not care if the rollingfileappender is active, just change the logfile in case // TODO this does not work - not even with @@ -537,7 +537,7 @@ public abstract class AbstractRipper } @Override - public abstract void setWorkingDir(URL url) throws IOException; + public abstract void setWorkingDir(URL url) throws IOException, URISyntaxException; /** * @@ -550,7 +550,7 @@ public abstract class AbstractRipper * @throws MalformedURLException * If any of those damned URLs gets malformed. */ - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { try { return getHost() + "_" + getGID(url); } catch (URISyntaxException e) { diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 12139cf3..27ad04e8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -192,7 +192,7 @@ public abstract class AlbumRipper extends AbstractRipper { * IOException */ @Override - public void setWorkingDir(URL url) throws IOException { + public void setWorkingDir(URL url) throws IOException, URISyntaxException { Path wd = Utils.getWorkingDirectory(); // TODO - change to nio String path = wd.toAbsolutePath().toString(); diff --git a/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java b/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java index 67572898..824d639e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java +++ b/src/main/java/com/rarchives/ripme/ripper/RipperInterface.java @@ -15,7 +15,7 @@ interface RipperInterface { void rip() throws IOException, URISyntaxException; boolean canRip(URL url); URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException; - void setWorkingDir(URL url) throws IOException; + void setWorkingDir(URL url) throws IOException, URISyntaxException; String getHost(); String getGID(URL url) throws MalformedURLException, URISyntaxException; } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/AerisdiesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/AerisdiesRipper.java index 4558f15e..8213d510 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/AerisdiesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/AerisdiesRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -46,7 +47,7 @@ public class AerisdiesRipper extends AbstractHTMLRipper { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { try { Element el = getCachedFirstPage().select(".headtext").first(); if (el == null) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java index 2d3194c6..8502e6b6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/BatoRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -69,7 +70,7 @@ public class BatoRipper extends AbstractHTMLRipper { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID return getHost() + "_" + getGID(url) + "_" + getCachedFirstPage().select("title").first().text().replaceAll(" ", "_"); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java index 875c0849..d86f63b8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java @@ -6,6 +6,7 @@ import com.rarchives.ripme.utils.Utils; import com.rarchives.ripme.utils.RipUtils; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; @@ -194,7 +195,7 @@ public class ChanRipper extends AbstractHTMLRipper { return this.url.getHost(); } - public Document getFirstPage() throws IOException { + public Document getFirstPage() throws IOException, URISyntaxException { return super.getFirstPage(); } private boolean isURLBlacklisted(String url) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java index 4cba64f7..c66465eb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; @@ -50,7 +51,7 @@ public class CheveretoRipper extends AbstractHTMLRipper { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID Element titleElement = getCachedFirstPage().select("meta[property=og:title]").first(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java index 33165c93..da9a0044 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java @@ -62,7 +62,7 @@ public class EHentaiRipper extends AbstractHTMLRipper { return "e-hentai.org"; } - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID if (albumDoc == null) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java index f60ced28..cede5274 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java @@ -57,7 +57,7 @@ public class EightmusesRipper extends AbstractHTMLRipper { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID Element titleElement = getCachedFirstPage().select("meta[name=description]").first(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java index d7b8015a..0f77e03c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EroShareRipper.java @@ -95,7 +95,7 @@ public class EroShareRipper extends AbstractHTMLRipper { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { if (!is_profile(url)) { try { // Attempt to use album title as GID diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java index bc1e9df9..3ad3f684 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; @@ -67,7 +68,7 @@ public class EromeRipper extends AbstractHTMLRipper { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID Element titleElement = getCachedFirstPage().select("meta[property=og:title]").first(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java index f33c079e..991a8ffb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java @@ -173,7 +173,7 @@ public class FlickrRipper extends AbstractHTMLRipper { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { if (!url.toExternalForm().contains("/sets/")) { return super.getAlbumTitle(url); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java index a5d14e98..49cbfc60 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -29,7 +30,7 @@ public class GirlsOfDesireRipper extends AbstractHTMLRipper { return "girlsofdesire.org"; } - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID Document doc = getCachedFirstPage(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HbrowseRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HbrowseRipper.java index 34072f22..040ca978 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HbrowseRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HbrowseRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -49,7 +50,7 @@ public class HbrowseRipper extends AbstractHTMLRipper { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { try { Document doc = getCachedFirstPage(); String title = doc.select("div[id=main] > table.listTable > tbody > tr > td.listLong").first().text(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java index cb521523..2b8ac967 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -90,7 +91,7 @@ public class Hentai2readRipper extends AbstractHTMLRipper { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { try { return getHost() + "_" + getGID(url); } catch (Exception e) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java index 086596a2..d6dba419 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -52,7 +53,7 @@ public class HentaifoxRipper extends AbstractHTMLRipper { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { try { Document doc = getCachedFirstPage(); String title = doc.select("div.info > h1").first().text(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java index 8d02ff56..d7c46a97 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -65,7 +66,7 @@ public class HitomiRipper extends AbstractHTMLRipper { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { try { // Attempt to use album title and username as GID Document doc = Http.url(url).get(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java index c8d383f5..fb071129 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java @@ -11,6 +11,7 @@ import org.jsoup.select.Elements; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -63,7 +64,7 @@ public class HqpornerRipper extends AbstractHTMLRipper { } @Override - public Document getFirstPage() throws IOException { + public Document getFirstPage() throws IOException, URISyntaxException { return super.getFirstPage(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index 9109466d..596680dd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -87,7 +87,7 @@ public class ImagebamRipper extends AbstractHTMLRipper { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID Elements elems = getCachedFirstPage().select("[id=gallery-name]"); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java index bcd5900f..5cfd3283 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java @@ -166,7 +166,7 @@ public class ImagefapRipper extends AbstractHTMLRipper { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID String title = getCachedFirstPage().title(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java index cbe401cb..deedfb88 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaicomicsRipper.java @@ -4,6 +4,7 @@ import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -82,7 +83,7 @@ public class MyhentaicomicsRipper extends AbstractHTMLRipper { } @Override - public Document getFirstPage() throws IOException { + public Document getFirstPage() throws IOException, URISyntaxException { return super.getFirstPage(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java index 595dd452..35a1f8ad 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java @@ -155,7 +155,7 @@ public class NfsfwRipper extends AbstractHTMLRipper { List imageURLs = getImagePageURLs(fstPage); List subalbumURLs = getSubalbumURLs(fstPage); return imageURLs.isEmpty() && !subalbumURLs.isEmpty(); - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { LOGGER.error("Unable to load " + url, e); return false; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java index 681738fa..c7245739 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Rule34Ripper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -51,13 +53,13 @@ public class Rule34Ripper extends AbstractHTMLRipper { "rule34.xxx/index.php?page=post&s=list&tags=TAG - got " + url + " instead"); } - public URL getAPIUrl() throws MalformedURLException { - URL urlToReturn = new URL("https://rule34.xxx/index.php?page=dapi&s=post&q=index&limit=100&tags=" + getGID(url)); + public URL getAPIUrl() throws MalformedURLException, URISyntaxException { + URL urlToReturn = new URI("https://rule34.xxx/index.php?page=dapi&s=post&q=index&limit=100&tags=" + getGID(url)).toURL(); return urlToReturn; } @Override - public Document getFirstPage() throws IOException { + public Document getFirstPage() throws IOException, URISyntaxException { apiUrl = getAPIUrl().toExternalForm(); // "url" is an instance field of the superclass return Http.url(getAPIUrl()).get(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java index 106d0a6d..ab9ebfa9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java @@ -43,7 +43,7 @@ public class SoundgasmRipper extends AbstractHTMLRipper { } @Override - public Document getFirstPage() throws IOException { + public Document getFirstPage() throws IOException, URISyntaxException { return super.getFirstPage(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java index fddbf1f2..72f65249 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ViewcomicRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -31,7 +32,7 @@ public class ViewcomicRipper extends AbstractHTMLRipper { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID String titleText = getCachedFirstPage().select("title").first().text(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java index d82f4aff..0da345b7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WebtoonsRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -43,7 +44,7 @@ public class WebtoonsRipper extends AbstractHTMLRipper { @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { Pattern pat = Pattern.compile("https?://www.webtoons.com/[a-zA-Z-_]+/[a-zA-Z_-]+/([a-zA-Z0-9_-]*)/[a-zA-Z0-9_-]+/\\S*"); Matcher mat = pat.matcher(url.toExternalForm()); if (mat.matches()) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java index 1019c7d8..6c696272 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/WordpressComicRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; @@ -220,7 +221,7 @@ public class WordpressComicRipper extends AbstractHTMLRipper { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { Pattern totempole666Pat = Pattern.compile("(?:https?://)?(?:www\\.)?totempole666.com/comic/([a-zA-Z0-9_-]*)/?$"); Matcher totempole666Mat = totempole666Pat.matcher(url.toExternalForm()); if (totempole666Mat.matches()) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 6da81338..da6e6f2c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -146,7 +147,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { } @Override - public Document getFirstPage() throws IOException { + public Document getFirstPage() throws IOException, URISyntaxException { return super.getFirstPage(); } @@ -215,7 +216,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { try { // Attempt to use album title and username as GID Document doc = getCachedFirstPage(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java index 5bcb89cf..ea19d484 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -104,7 +105,7 @@ public class XvideosRipper extends AbstractSingleFileRipper { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/profiles/([a-zA-Z0-9_-]+)/photos/(\\d+)/([a-zA-Z0-9_-]+)$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java index 5d95580b..043d1835 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ZizkiRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; @@ -45,7 +46,7 @@ public class ZizkiRipper extends AbstractHTMLRipper { } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { try { // Attempt to use album title as GID Element titleElement = getCachedFirstPage().select("h1.title").first(); diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index d2ff250a..e5cc2f88 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1502,7 +1502,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { entry.count = rsc.count; try { entry.title = ripper.getAlbumTitle(ripper.getURL()); - } catch (MalformedURLException e) { + } catch (MalformedURLException | URISyntaxException e) { LOGGER.warn(e.getMessage()); } HISTORY.add(entry); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java index 016ba218..ed023d47 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java @@ -23,7 +23,7 @@ import org.junit.jupiter.api.Test; public class ChanRipperTest extends RippersTest { @Test @Tag("flaky") - public void testChanURLPasses() throws IOException, URISyntaxException { + public void testChanURLPasses() throws IOException, URISyntaxException { List passURLs = new ArrayList<>(); // URLs that should work passURLs.add(new URI("http://desuchan.net/v/res/7034.html").toURL()); @@ -37,7 +37,7 @@ public class ChanRipperTest extends RippersTest { CompletableFuture setupFuture = CompletableFuture.runAsync(() -> { try { ripper.setup(); - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { throw new RuntimeException(e); } }); From e5c367df443d17c7394b9aeb92050b461f71e509 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 29 Dec 2023 11:40:54 +0100 Subject: [PATCH 453/512] new URI instead of new URL, jpg3, hentaidude. --- .../com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java | 3 ++- .../java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java | 5 +++-- .../ripme/tst/ripper/rippers/HentaidudeRipperTest.java | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java index 25bc57d3..24625859 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java @@ -10,6 +10,7 @@ import org.jsoup.nodes.Document; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; @@ -91,7 +92,7 @@ public class HentaidudeRipper extends AbstractSingleFileRipper { public void run() { try { Document doc = Http.url(url).get(); - URL videoSourceUrl = new URL(getVideoUrl(doc)); + URL videoSourceUrl = new URI(getVideoUrl(doc)).toURL(); addURLToDownload(videoSourceUrl, "", "", "", null, getVideoName(), "mp4"); } catch (Exception e) { LOGGER.error("Could not get video url for " + getVideoName(), e); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java index 6a8cd5b5..c79e02bc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java @@ -7,6 +7,7 @@ import org.jsoup.nodes.Element; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -40,11 +41,11 @@ public class Jpg3Ripper extends AbstractHTMLRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { String u = url.toExternalForm(); u = u.replaceAll("https?://jpg3.su/a/([^/]+)/?.*", "https://jpg3.su/a/$1"); LOGGER.debug("Changed URL from " + url + " to " + u); - return new URL(u); + return new URI(u).toURL(); } @Override diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaidudeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaidudeRipperTest.java index 3ac9f4bf..0283f9b7 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaidudeRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaidudeRipperTest.java @@ -1,6 +1,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.HentaidudeRipper; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.net.URI; @@ -8,6 +9,7 @@ import java.net.URISyntaxException; public class HentaidudeRipperTest extends RippersTest{ + @Test public void testRip() throws IOException, URISyntaxException { HentaidudeRipper ripper = new HentaidudeRipper(new URI("https://hentaidude.com/girlfriends-4ever-dlc-2/").toURL()); testRipper(ripper); From 7afe1eb7d9322211f3171eb65bcf715e8ae7e4a3 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 29 Dec 2023 11:49:08 +0100 Subject: [PATCH 454/512] new URI instead of new URL, deviantart, flickr. --- .../ripme/ripper/rippers/DeviantartRipper.java | 14 +++++++------- .../ripme/ripper/rippers/FlickrRipper.java | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 3fc34ef3..9f26a268 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -13,6 +13,8 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.Serializable; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; @@ -381,11 +383,11 @@ public class DeviantartRipper extends AbstractHTMLRipper { try { String url = cleanURL(); if (this.usingCatPath) { - return (new URL(url + "?catpath=/&offset=" + offset)); + return (new URI(url + "?catpath=/&offset=" + offset)).toURL(); } else { - return (new URL(url + "?offset=" + offset)); + return (new URI(url + "?offset=" + offset).toURL()); } - } catch (MalformedURLException e) { + } catch (MalformedURLException | URISyntaxException e) { e.printStackTrace(); } return null; @@ -531,8 +533,6 @@ public class DeviantartRipper extends AbstractHTMLRipper { /** * Get URL to Artwork and return fullsize URL with file ending. * - * @param page Like - * https://www.deviantart.com/apofiss/art/warmest-of-the-days-455668450 * @return URL like * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/07f7a6bb-2d35-4630-93fc-be249af22b3e/d7jak0y-d20e5932-df72-4d13-b002-5e122037b373.jpg * @@ -628,11 +628,11 @@ public class DeviantartRipper extends AbstractHTMLRipper { } String[] tmpParts = downloadString.split("\\."); //split to get file ending - addURLToDownload(new URL(downloadString), "", "", "", new HashMap(), + addURLToDownload(new URI(downloadString).toURL(), "", "", "", new HashMap(), title + "." + tmpParts[tmpParts.length - 1]); return; - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { e.printStackTrace(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java index 991a8ffb..4623df98 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java @@ -262,7 +262,7 @@ public class FlickrRipper extends AbstractHTMLRipper { JSONObject data = (JSONObject) pictures.get(i); try { addURLToDownload(getLargestImageURL(data.getString("id"), apiKey)); - } catch (MalformedURLException e) { + } catch (MalformedURLException | URISyntaxException e) { LOGGER.error("Flickr MalformedURLException: " + e.getMessage()); } @@ -285,11 +285,11 @@ public class FlickrRipper extends AbstractHTMLRipper { addURLToDownload(url, getPrefix(index)); } - private URL getLargestImageURL(String imageID, String apiKey) throws MalformedURLException { + private URL getLargestImageURL(String imageID, String apiKey) throws MalformedURLException, URISyntaxException { TreeMap imageURLMap = new TreeMap<>(); try { - URL imageAPIURL = new URL("https://www.flickr.com/services/rest/?method=flickr.photos.getSizes&api_key=" + apiKey + "&photo_id=" + imageID + "&format=json&nojsoncallback=1"); + URL imageAPIURL = new URI("https://www.flickr.com/services/rest/?method=flickr.photos.getSizes&api_key=" + apiKey + "&photo_id=" + imageID + "&format=json&nojsoncallback=1").toURL(); JSONArray imageSizes = new JSONObject(Http.url(imageAPIURL).ignoreContentType().get().text()).getJSONObject("sizes").getJSONArray("size"); for (int i = 0; i < imageSizes.length(); i++) { JSONObject imageInfo = imageSizes.getJSONObject(i); @@ -304,6 +304,6 @@ public class FlickrRipper extends AbstractHTMLRipper { LOGGER.error("IOException while looking at image sizes: " + e.getMessage()); } - return new URL(imageURLMap.lastEntry().getValue()); + return new URI(imageURLMap.lastEntry().getValue()).toURL(); } } From 52425a7ad70f186cc825432bc10c8de2c9f4ab86 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 29 Dec 2023 11:56:08 +0100 Subject: [PATCH 455/512] release 2.1.8 --- ripme.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ripme.json b/ripme.json index e856cbaa..b226b653 100644 --- a/ripme.json +++ b/ripme.json @@ -1,7 +1,8 @@ { - "latestVersion": "2.1.7-29-b080faae", - "currentHash": "a2bedd4d524efd98884dad38c888059ea5bbc113924eefa73f5893189beab075", + "latestVersion": "2.1.8-1-f5153de8", + "currentHash": "a23964a55dfaac7108552d27d964d52ee2b8404c18c6e0904c681c679951ba24", "changeList": [ + "2.1.8-1-f5153de8: jpg3 add, java-21 adjustments.", "2.1.7-29-b080faae: luciousripper fix, java-21 adjustments.", "2.1.6-1-68189f27: erome fix.", "2.1.5-8-ba51d7b: ripme running with java-17.", From d1166a73bfa0ef8cf0f9d2ac23518d76987b9862 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 29 Dec 2023 12:01:22 +0100 Subject: [PATCH 456/512] new URI instead of new URL, flickr, hqporn. --- .../com/rarchives/ripme/ripper/rippers/FlickrRipper.java | 4 ++-- .../rarchives/ripme/ripper/rippers/HqpornerRipper.java | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java index 4623df98..c58a7e71 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java @@ -130,8 +130,8 @@ public class FlickrRipper extends AbstractHTMLRipper { String apiURL = null; try { apiURL = apiURLBuilder(getAlbum(url.toExternalForm()), page, apiKey); - pageURL = new URL(apiURL); - } catch (MalformedURLException e) { + pageURL = new URI(apiURL).toURL(); + } catch (MalformedURLException | URISyntaxException e) { LOGGER.error("Unable to get api link " + apiURL + " is malformed"); } try { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java index fb071129..0f69c75b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java @@ -11,6 +11,7 @@ import org.jsoup.select.Elements; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -164,10 +165,10 @@ public class HqpornerRipper extends AbstractHTMLRipper { } if (downloadUrl != null) { - addURLToDownload(new URL(downloadUrl), "", subdirectory, "", null, getVideoName(), "mp4"); + addURLToDownload(new URI(downloadUrl).toURL(), "", subdirectory, "", null, getVideoName(), "mp4"); } - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { LOGGER.error("[!] Exception while downloading video.", e); } } @@ -215,7 +216,7 @@ public class HqpornerRipper extends AbstractHTMLRipper { try { logger.info("Trying to download from unknown video host " + videoPageurl); - URL url = new URL(videoPageurl); + URL url = new URI(videoPageurl).toURL(); Response response = Http.url(url).referrer(hqpornerVideoPageUrl).response(); Document doc = response.parse(); @@ -245,7 +246,7 @@ public class HqpornerRipper extends AbstractHTMLRipper { } } - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { logger.error("Unable to get video url using generic methods."); } From 65f0f0e562eb9d1c7e93ed167ac82bb043a4dc77 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 29 Dec 2023 12:16:08 +0100 Subject: [PATCH 457/512] new URI instead of new URL, E621. --- .../com/rarchives/ripme/ripper/rippers/E621Ripper.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java index 13f75f22..1d29a736 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java @@ -10,6 +10,8 @@ import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; @@ -181,14 +183,14 @@ public class E621Ripper extends AbstractHTMLRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { if (gidPattern2 == null) gidPattern2 = Pattern.compile( "^https?://(www\\.)?e621\\.net/post/search\\?tags=([a-zA-Z0-9$_.+!*'():,%-]+)(/.*)?(#.*)?$"); Matcher m = gidPattern2.matcher(url.toExternalForm()); if (m.matches()) - return new URL("https://e621.net/post/index/1/" + m.group(2).replace("+", "%20")); + return new URI("https://e621.net/post/index/1/" + m.group(2).replace("+", "%20")).toURL(); return url; } @@ -208,9 +210,9 @@ public class E621Ripper extends AbstractHTMLRipper { try { String fullSizedImage = getFullSizedImage(url); if (fullSizedImage != null && !fullSizedImage.equals("")) { - addURLToDownload(new URL(fullSizedImage), index); + addURLToDownload(new URI(fullSizedImage).toURL(), index); } - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { logger.error("Unable to get full sized image from " + url); } } From f74c72783bd757d681b2c05cf7dc664c2284ecb7 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 29 Dec 2023 12:19:41 +0100 Subject: [PATCH 458/512] new URI instead of new URL, tumblr. --- .../ripme/ripper/rippers/TumblrRipper.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java index 0c561d77..6d91361b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java @@ -1,9 +1,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.MalformedURLException; -import java.net.URL; +import java.net.*; import java.util.Arrays; import java.util.List; import java.util.Random; @@ -100,11 +98,11 @@ public class TumblrRipper extends AlbumRipper { * @throws MalformedURLException */ @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { String u = url.toExternalForm(); // Convert .tumblr.com/path to /path if needed if (StringUtils.countMatches(u, ".") > 2) { - url = new URL(u.replace(".tumblr.com", "")); + url = new URI(u.replace(".tumblr.com", "")).toURL(); if (isTumblrURL(url)) { LOGGER.info("Detected tumblr site: " + url); } @@ -263,7 +261,7 @@ public class TumblrRipper extends AlbumRipper { fileLocation = photo.getJSONObject("original_size").getString("url").replaceAll("http:", "https:"); qualM = qualP.matcher(fileLocation); fileLocation = qualM.replaceFirst("_1280.$1"); - fileURL = new URL(fileLocation); + fileURL = new URI(fileLocation).toURL(); m = p.matcher(fileURL.toString()); if (m.matches()) { @@ -278,7 +276,7 @@ public class TumblrRipper extends AlbumRipper { } } else if (post.has("video_url")) { try { - fileURL = new URL(post.getString("video_url").replaceAll("http:", "https:")); + fileURL = new URI(post.getString("video_url").replaceAll("http:", "https:")).toURL(); downloadURL(fileURL, date); } catch (Exception e) { LOGGER.error("[!] Error while parsing video in " + post, e); @@ -293,8 +291,8 @@ public class TumblrRipper extends AlbumRipper { // If the image is any smaller, it will still get the largest available size qualM = qualP.matcher(imgSrc); imgSrc = qualM.replaceFirst("_1280.$1"); - downloadURL(new URL(imgSrc), date); - } catch (MalformedURLException e) { + downloadURL(new URI(imgSrc).toURL(), date); + } catch (MalformedURLException | URISyntaxException e) { LOGGER.error("[!] Error while getting embedded image at " + post, e); return true; } From eb351d4b0586bc48ef55a07f009959901b78c63e Mon Sep 17 00:00:00 2001 From: joroto <47276635+joroto@users.noreply.github.com> Date: Mon, 11 Mar 2024 08:26:21 +0200 Subject: [PATCH 459/512] EightmusesRipper fixed (#169) * EightmusesRipper fixed --- .../ripper/rippers/EightmusesRipper.java | 55 +++---------------- 1 file changed, 9 insertions(+), 46 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java index cede5274..7cfd568f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java @@ -1,10 +1,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; +import java.net.*; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -12,8 +9,6 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.rarchives.ripme.utils.Utils; -import org.json.JSONObject; import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -25,7 +20,7 @@ import com.rarchives.ripme.utils.Http; public class EightmusesRipper extends AbstractHTMLRipper { - private Map cookies = new HashMap<>(); + private Map cookies = new HashMap<>(); public EightmusesRipper(URL url) throws IOException { super(url); @@ -82,10 +77,10 @@ public class EightmusesRipper extends AbstractHTMLRipper { @Override public List getURLsFromPage(Document page) { List imageURLs = new ArrayList<>(); - int x = 1; // This contains the thumbnails of all images on the page Elements pageImages = page.getElementsByClass("c-tile"); - for (Element thumb : pageImages) { + for (int i = 0; i < pageImages.size(); i++) { + Element thumb = pageImages.get(i); // If true this link is a sub album if (thumb.attr("href").contains("/comics/album/")) { String subUrl = "https://www.8muses.com" + thumb.attr("href"); @@ -109,24 +104,14 @@ public class EightmusesRipper extends AbstractHTMLRipper { if (thumb.hasAttr("data-cfsrc")) { image = thumb.attr("data-cfsrc"); } else { - // Deobfustace the json data - String rawJson = deobfuscateJSON(page.select("script#ractive-public").html() - .replaceAll(">", ">").replaceAll("<", "<").replace("&", "&")); - JSONObject json = new JSONObject(rawJson); + Element imageElement = thumb.select("img").first(); + image = "https://comics.8muses.com" + imageElement.attr("data-src").replace("/th/", "/fl/"); try { - for (int i = 0; i != json.getJSONArray("pictures").length(); i++) { - image = "https://www.8muses.com/image/fl/" + json.getJSONArray("pictures").getJSONObject(i).getString("publicUri"); - URL imageUrl = new URI(image).toURL(); - addURLToDownload(imageUrl, getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, getPrefixShort(x), "", null, true); - // X is our page index - x++; - if (isThisATest()) { - break; - } - } - return imageURLs; + URL imageUrl = new URI(image).toURL(); + addURLToDownload(imageUrl, getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, getPrefixShort(i), "", null, true); } catch (MalformedURLException | URISyntaxException e) { LOGGER.error("\"" + image + "\" is malformed"); + LOGGER.error(e.getMessage()); } } if (!image.contains("8muses.com")) { @@ -166,26 +151,4 @@ public class EightmusesRipper extends AbstractHTMLRipper { public String getPrefixShort(int index) { return String.format("%03d", index); } - - private String deobfuscateJSON(String obfuscatedString) { - StringBuilder deobfuscatedString = new StringBuilder(); - // The first char in one of 8muses obfuscated strings is always ! so we replace it - for (char ch : obfuscatedString.replaceFirst("!", "").toCharArray()){ - deobfuscatedString.append(deobfuscateChar(ch)); - } - return deobfuscatedString.toString(); - } - - private String deobfuscateChar(char c) { - if ((int) c == 32) { - return fromCharCode(32); - } else if ((int) c > 120){ - return fromCharCode((int)c); - } - return fromCharCode(33 + (c + 14) % 94); - } - - private static String fromCharCode(int... codePoints) { - return new String(codePoints, 0, codePoints.length); - } } \ No newline at end of file From ab2b27c6a6a372fc6ecfdf5b43f7133ff87c9346 Mon Sep 17 00:00:00 2001 From: joroto <47276635+joroto@users.noreply.github.com> Date: Mon, 11 Mar 2024 08:27:58 +0200 Subject: [PATCH 460/512] Imagefap: add retry logic for getFullSizedImage() (#181) --- .../ripme/ripper/rippers/ImagefapRipper.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java index 5cfd3283..4fcf2201 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java @@ -144,8 +144,17 @@ public class ImagefapRipper extends AbstractHTMLRipper { } String image = getFullSizedImage("https://www.imagefap.com" + thumb.parent().attr("href")); - if(image == null) - throw new RuntimeException("Unable to extract image URL from single image page! Unable to continue"); + if (image == null) { + for (int i = 0; i < HTTP_RETRY_LIMIT; i++) { + image = getFullSizedImage("https://www.imagefap.com" + thumb.parent().attr("href")); + if (image != null) { + break; + } + sleep(PAGE_SLEEP_TIME); + } + if (image == null) + throw new RuntimeException("Unable to extract image URL from single image page! Unable to continue"); + } LOGGER.debug("Adding imageURL: '" + image + "'"); From ad86ae11daa64a1bdf0f4feb1e434d96d39f4ae0 Mon Sep 17 00:00:00 2001 From: joroto <47276635+joroto@users.noreply.github.com> Date: Mon, 11 Mar 2024 08:31:00 +0200 Subject: [PATCH 461/512] HistoryMenuMouseListener right click menu fix (#174) --- .../rarchives/ripme/ui/HistoryMenuMouseListener.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java b/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java index cf288f2d..8a69477c 100644 --- a/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java +++ b/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java @@ -62,7 +62,16 @@ class HistoryMenuMouseListener extends MouseAdapter { } @Override - public void mouseClicked(MouseEvent e) { + public void mousePressed(MouseEvent e) { + checkPopupTrigger(e); + } + + @Override + public void mouseReleased(MouseEvent e) { + checkPopupTrigger(e); + } + + private void checkPopupTrigger(MouseEvent e) { if (e.getModifiersEx() == InputEvent.BUTTON3_DOWN_MASK) { if (!(e.getSource() instanceof JTable)) { return; From 22e915df001867911cf002ebc99d7f3c500c443d Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 11 Mar 2024 07:40:41 +0100 Subject: [PATCH 462/512] release 2.1.9 --- ripme.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ripme.json b/ripme.json index b226b653..97489987 100644 --- a/ripme.json +++ b/ripme.json @@ -1,7 +1,8 @@ { - "latestVersion": "2.1.8-1-f5153de8", - "currentHash": "a23964a55dfaac7108552d27d964d52ee2b8404c18c6e0904c681c679951ba24", + "latestVersion": "2.1.9-6-ad86ae11", + "currentHash": "6ed316f7cdf979cd6dc563b02b5668feae2ba6a6d12c8b8991e6911eb3289afd", "changeList": [ + "2.1.9-6-ad86ae11, HistoryMenuMouseListener right click menu, Imagefap retry logic for getFullSizedImage(), EightmusesRipper fixed", "2.1.8-1-f5153de8: jpg3 add, java-21 adjustments.", "2.1.7-29-b080faae: luciousripper fix, java-21 adjustments.", "2.1.6-1-68189f27: erome fix.", From e8052d086f9d48c144b34d569fb8e34c55e75ad2 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 18 Mar 2024 03:26:35 +0100 Subject: [PATCH 463/512] release 2.1.9, this time also the binary is on github --- ripme.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ripme.json b/ripme.json index 97489987..c2f54680 100644 --- a/ripme.json +++ b/ripme.json @@ -1,8 +1,8 @@ { - "latestVersion": "2.1.9-6-ad86ae11", - "currentHash": "6ed316f7cdf979cd6dc563b02b5668feae2ba6a6d12c8b8991e6911eb3289afd", + "latestVersion": "2.1.9-7-22e915df", + "currentHash": "782ffec29bd14cfde6d714fa6f76980b3fd7cf96723b1121976134a6a5057e68", "changeList": [ - "2.1.9-6-ad86ae11, HistoryMenuMouseListener right click menu, Imagefap retry logic for getFullSizedImage(), EightmusesRipper fixed", + "2.1.9-7-22e915df, HistoryMenuMouseListener right click menu, Imagefap retry logic for getFullSizedImage(), EightmusesRipper fixed", "2.1.8-1-f5153de8: jpg3 add, java-21 adjustments.", "2.1.7-29-b080faae: luciousripper fix, java-21 adjustments.", "2.1.6-1-68189f27: erome fix.", From 404da9b5e5091a825e864e18c1e3822d28fc73f3 Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 18 Mar 2024 09:11:02 +0100 Subject: [PATCH 464/512] java-22, java-21 default --- .github/workflows/gradle.yml | 6 +++--- build.gradle.kts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 0a1e892a..675cda3e 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -15,10 +15,10 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - java: [21] + java: [22] include: # test old java on one os only, upload from ubuntu java-17 - os: ubuntu-latest - java: 17 + java: 21 upload: true steps: @@ -30,7 +30,7 @@ jobs: uses: FranzDiebold/github-env-vars-action@v2 - name: Set up java - uses: actions/setup-java@v3.13.0 + uses: actions/setup-java@v4.2.1 with: java-version: ${{ matrix.java }} distribution: zulu diff --git a/build.gradle.kts b/build.gradle.kts index 3266fae6..30f93110 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -4,7 +4,7 @@ // gradle clean build -PjavacRelease=21 // gradle clean build -PcustomVersion=1.0.0-10-asdf val customVersion = (project.findProperty("customVersion") ?: "") as String -val javacRelease = (project.findProperty("javacRelease") ?: "17") as String +val javacRelease = (project.findProperty("javacRelease") ?: "21") as String plugins { id("fr.brouillard.oss.gradle.jgitver") version "0.9.1" From c2d1472008051204d247b5e1cb4ca7f7021d7782 Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 20 Mar 2024 12:33:17 +0100 Subject: [PATCH 465/512] xhamster.one domain not exists as before. --- .../ripme/ripper/rippers/XhamsterRipper.java | 20 +++++++++---------- .../ripper/rippers/XhamsterRipperTest.java | 6 ------ 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index da6e6f2c..4c62a184 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -57,17 +57,17 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://([\\w\\w]*\\.)?xhamster([^<]*)\\.(com|one|desi)/photos/gallery/.*?(\\d+)$"); + Pattern p = Pattern.compile("^https?://([\\w\\w]*\\.)?xhamster([^<]*)\\.(com|desi)/photos/gallery/.*?(\\d+)$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(4); } - p = Pattern.compile("^https?://[\\w\\w.]*xhamster([^<]*)\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); + p = Pattern.compile("^https?://[\\w\\w.]*xhamster([^<]*)\\.(com|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return "user_" + m.group(1); } - p = Pattern.compile("^https?://.*xhamster([^<]*)\\.(com|one|desi)/(movies|videos)/(.*$)"); + p = Pattern.compile("^https?://.*xhamster([^<]*)\\.(com|desi)/(movies|videos)/(.*$)"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(4); @@ -100,7 +100,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public boolean pageContainsAlbums(URL url) { - Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster([^<]*)\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); + Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster([^<]*)\\.(com|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); Matcher m = p.matcher(url.toExternalForm()); LOGGER.info("Checking if page has albums"); LOGGER.info(m.matches()); @@ -109,17 +109,17 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public boolean canRip(URL url) { - Pattern p = Pattern.compile("^https?://([\\w\\w]*\\.)?xhamster([^<]*)\\.(com|one|desi)/photos/gallery/.*?(\\d+)$"); + Pattern p = Pattern.compile("^https?://([\\w\\w]*\\.)?xhamster([^<]*)\\.(com|desi)/photos/gallery/.*?(\\d+)$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return true; } - p = Pattern.compile("^https?://[\\w\\w.]*xhamster([^<]*)\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); + p = Pattern.compile("^https?://[\\w\\w.]*xhamster([^<]*)\\.(com|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return true; } - p = Pattern.compile("^https?://.*xhamster([^<]*)\\.(com|one|desi)/(movies|videos)/(.*$)"); + p = Pattern.compile("^https?://.*xhamster([^<]*)\\.(com|desi)/(movies|videos)/(.*$)"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return true; @@ -128,7 +128,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { } private boolean isVideoUrl(URL url) { - Pattern p = Pattern.compile("^https?://.*xhamster([^<]*)\\.(com|one|desi)/(movies|videos)/(.*$)"); + Pattern p = Pattern.compile("^https?://.*xhamster([^<]*)\\.(com|desi)/(movies|videos)/(.*$)"); Matcher m = p.matcher(url.toExternalForm()); return m.matches(); } @@ -208,9 +208,9 @@ public class XhamsterRipper extends AbstractHTMLRipper { private void downloadFile(String url) { try { - addURLToDownload(new URL(url), getPrefix(index)); + addURLToDownload(new URI(url).toURL(), getPrefix(index)); index = index + 1; - } catch (MalformedURLException e) { + } catch (MalformedURLException | URISyntaxException e) { LOGGER.error("The url \"" + url + "\" is malformed"); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java index 68747ea4..24555e89 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java @@ -33,12 +33,6 @@ public class XhamsterRipperTest extends RippersTest { } @Test @Tag("flaky") - public void testXhamsterAlbumOneDomain() throws IOException, URISyntaxException { - XhamsterRipper ripper = new XhamsterRipper(new URI("https://xhamster.one/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664").toURL()); - testRipper(ripper); - } - @Test - @Tag("flaky") public void testXhamsterAlbumDesiDomain() throws IOException, URISyntaxException { XhamsterRipper ripper = new XhamsterRipper(new URI("https://xhamster5.desi/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664").toURL()); testRipper(ripper); From bce4ddd74d528e9b3ff10df79f7c89723b30b6f5 Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 20 Mar 2024 08:54:52 +0100 Subject: [PATCH 466/512] new URI instead of new URL. --- .../ripper/rippers/ArtStationRipper.java | 20 +++++++------ .../ripme/ripper/rippers/ChanRipper.java | 2 +- .../ripme/ripper/rippers/EHentaiRipper.java | 13 ++------- .../ripme/ripper/rippers/EromeRipper.java | 5 ++-- .../ripper/rippers/FivehundredpxRipper.java | 29 +++++++------------ .../ripme/ripper/rippers/FuskatorRipper.java | 6 ++-- .../ripper/rippers/HentaiNexusRipper.java | 11 +++---- .../ripme/ripper/rippers/HitomiRipper.java | 6 ++-- .../ripme/ripper/rippers/NsfwXxxRipper.java | 18 +++++++----- .../ripme/ripper/rippers/XhamsterRipper.java | 9 +++--- .../rippers/video/CliphunterRipper.java | 6 ++-- .../ripper/rippers/video/PornhubRipper.java | 6 ++-- .../ripper/rippers/video/StickyXXXRipper.java | 11 +++---- .../rippers/video/TwitchVideoRipper.java | 11 +++---- .../ripper/rippers/video/ViddmeRipper.java | 6 ++-- .../ripper/rippers/video/VidearnRipper.java | 13 ++++----- .../ripme/ripper/rippers/video/VkRipper.java | 6 ++-- .../ripper/rippers/StickyXXXRipperTest.java | 2 ++ 18 files changed, 88 insertions(+), 92 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java index 66455861..bc824769 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -50,7 +52,7 @@ public class ArtStationRipper extends AbstractJSONRipper { try { // groupData = Http.url(albumURL.getLocation()).getJSON(); groupData = getJson(albumURL.getLocation()); - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { throw new MalformedURLException("Couldn't load JSON from " + albumURL.getLocation()); } return groupData.getString("title"); @@ -62,7 +64,7 @@ public class ArtStationRipper extends AbstractJSONRipper { try { // groupData = Http.url(userInfoURL).getJSON(); groupData = getJson(userInfoURL); - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { throw new MalformedURLException("Couldn't load JSON from " + userInfoURL); } return groupData.getString("full_name"); @@ -74,7 +76,7 @@ public class ArtStationRipper extends AbstractJSONRipper { } @Override - protected JSONObject getFirstPage() throws IOException { + protected JSONObject getFirstPage() throws IOException, URISyntaxException { if (albumURL.getType() == URL_TYPE.SINGLE_PROJECT) { // URL points to JSON of a single project, just return it // return Http.url(albumURL.getLocation()).getJSON(); @@ -90,7 +92,7 @@ public class ArtStationRipper extends AbstractJSONRipper { if (albumContent.getInt("total_count") > 0) { // Get JSON of the first project and return it JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(0); - ParsedURL projectURL = parseURL(new URL(projectInfo.getString("permalink"))); + ParsedURL projectURL = parseURL(new URI(projectInfo.getString("permalink")).toURL()); // return Http.url(projectURL.getLocation()).getJSON(); return getJson(projectURL.getLocation()); } @@ -100,7 +102,7 @@ public class ArtStationRipper extends AbstractJSONRipper { } @Override - protected JSONObject getNextPage(JSONObject doc) throws IOException { + protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException { if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) { // Initialize the page number if it hasn't been initialized already if (projectPageNumber == null) { @@ -117,7 +119,7 @@ public class ArtStationRipper extends AbstractJSONRipper { projectIndex = 0; } - Integer currentProject = ((projectPageNumber - 1) * 50) + (projectIndex + 1); + int currentProject = ((projectPageNumber - 1) * 50) + (projectIndex + 1); // JSONObject albumContent = Http.url(albumURL.getLocation() + "?page=" + // projectPageNumber).getJSON(); JSONObject albumContent = getJson(albumURL.getLocation() + "?page=" + projectPageNumber); @@ -125,7 +127,7 @@ public class ArtStationRipper extends AbstractJSONRipper { if (albumContent.getInt("total_count") > currentProject) { // Get JSON of the next project and return it JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(projectIndex); - ParsedURL projectURL = parseURL(new URL(projectInfo.getString("permalink"))); + ParsedURL projectURL = parseURL(new URI(projectInfo.getString("permalink")).toURL()); projectIndex++; // return Http.url(projectURL.getLocation()).getJSON(); return getJson(projectURL.getLocation()); @@ -320,8 +322,8 @@ public class ArtStationRipper extends AbstractJSONRipper { throw new IOException("Error fetching json. Status code:" + status); } - private JSONObject getJson(String url) throws IOException { - return getJson(new URL(url)); + private JSONObject getJson(String url) throws IOException, URISyntaxException { + return getJson(new URI(url).toURL()); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java index d86f63b8..f1d41426 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java @@ -72,7 +72,7 @@ public class ChanRipper extends AbstractHTMLRipper { ); private ChanSite chanSite; - private Boolean generalChanSite = true; + private boolean generalChanSite = true; public ChanRipper(URL url) throws IOException { super(url); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java index da9a0044..81f09aa4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java @@ -94,13 +94,6 @@ public class EHentaiRipper extends AbstractHTMLRipper { + " Got: " + url); } - /** - * Attempts to get page, checks for IP ban, waits. - * - * @param url - * @return Page document - * @throws IOException If page loading errors, or if retries are exhausted - */ private Document getPageWithRetries(URL url) throws IOException { Document doc; int retries = 3; @@ -251,16 +244,16 @@ public class EHentaiRipper extends AbstractHTMLRipper { savePath += String.format("%03d_", index); } savePath += m.group(1); - addURLToDownload(new URL(imgsrc), Paths.get(savePath)); + addURLToDownload(new URI(imgsrc).toURL(), Paths.get(savePath)); } else { // Provide prefix and let the AbstractRipper "guess" the filename String prefix = ""; if (Utils.getConfigBoolean("download.save_order", true)) { prefix = String.format("%03d_", index); } - addURLToDownload(new URL(imgsrc), prefix); + addURLToDownload(new URI(imgsrc).toURL(), prefix); } - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { LOGGER.error("[!] Exception while loading/parsing " + this.url, e); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java index 3ad3f684..3035d746 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -85,8 +86,8 @@ public class EromeRipper extends AbstractHTMLRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return new URL(url.toExternalForm().replaceAll("https?://erome.com", "https://www.erome.com")); + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { + return new URI(url.toExternalForm().replaceAll("https?://erome.com", "https://www.erome.com")).toURL(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java index 79edab1c..bba284f1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java @@ -1,9 +1,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.MalformedURLException; -import java.net.URL; +import java.net.*; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -164,8 +162,8 @@ public class FivehundredpxRipper extends AbstractJSONRipper { } @Override - public JSONObject getFirstPage() throws IOException { - URL apiURL = new URL(baseURL + "&consumer_key=" + CONSUMER_KEY); + public JSONObject getFirstPage() throws IOException, URISyntaxException { + URL apiURL = new URI(baseURL + "&consumer_key=" + CONSUMER_KEY).toURL(); LOGGER.debug("apiURL: " + apiURL); JSONObject json = Http.url(apiURL).getJSON(); @@ -232,7 +230,7 @@ public class FivehundredpxRipper extends AbstractJSONRipper { } @Override - public JSONObject getNextPage(JSONObject json) throws IOException { + public JSONObject getNextPage(JSONObject json) throws IOException, URISyntaxException { if (isThisATest()) { return null; } @@ -249,9 +247,9 @@ public class FivehundredpxRipper extends AbstractJSONRipper { sleep(500); ++page; - URL apiURL = new URL(baseURL + URL apiURL = new URI(baseURL + "&page=" + page - + "&consumer_key=" + CONSUMER_KEY); + + "&consumer_key=" + CONSUMER_KEY).toURL(); return Http.url(apiURL).getJSON(); } @@ -296,14 +294,9 @@ public class FivehundredpxRipper extends AbstractJSONRipper { } } } - if (imageURL == null) { - LOGGER.error("Failed to find image for photo " + photo.toString()); - } - else { - imageURLs.add(imageURL); - if (isThisATest()) { - break; - } + imageURLs.add(imageURL); + if (isThisATest()) { + break; } } return imageURLs; @@ -311,13 +304,13 @@ public class FivehundredpxRipper extends AbstractJSONRipper { private boolean urlExists(String url) { try { - HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); + HttpURLConnection connection = (HttpURLConnection) new URI(url).toURL().openConnection(); connection.setRequestMethod("HEAD"); if (connection.getResponseCode() != 200) { throw new IOException("Couldn't find full-size image at " + url); } return true; - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { return false; } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java index d88b16e8..62a60fcc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -40,7 +42,7 @@ public class FuskatorRipper extends AbstractHTMLRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { String u = url.toExternalForm(); if (u.contains("/thumbs/")) { u = u.replace("/thumbs/", "/full/"); @@ -48,7 +50,7 @@ public class FuskatorRipper extends AbstractHTMLRipper { if (u.contains("/expanded/")) { u = u.replaceAll("/expanded/", "/full/"); } - return new URL(u); + return new URI(u).toURL(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java index ca709418..4d28f7a2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.Base64; @@ -80,16 +82,15 @@ public class HentaiNexusRipper extends AbstractJSONRipper { } @Override - protected JSONObject getFirstPage() throws IOException { + protected JSONObject getFirstPage() throws IOException, URISyntaxException { String jsonEncodedString = getJsonEncodedStringFromPage(); String jsonDecodedString = decodeJsonString(jsonEncodedString); return new JSONObject(jsonDecodedString); } - public String getJsonEncodedStringFromPage() throws MalformedURLException, IOException - { + public String getJsonEncodedStringFromPage() throws MalformedURLException, IOException, URISyntaxException { // Image data only appears on the /read/ page and not on the /view/ one. - URL readUrl = new URL(String.format("http://hentainexus.com/read/%s",getGID(url))); + URL readUrl = new URI(String.format("http://hentainexus.com/read/%s",getGID(url))).toURL(); Document document = Http.url(readUrl).response().parse(); for (Element scripts : document.getElementsByTag("script")) { @@ -143,7 +144,7 @@ public class HentaiNexusRipper extends AbstractJSONRipper { } magicByte = (byte) (magicByte & 0x7); - ArrayList newArray = new ArrayList(); + ArrayList newArray = new ArrayList<>(); for (int i = 0x0; i < 0x100; i++) { newArray.add(i); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java index d7c46a97..d312b75b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -14,7 +15,6 @@ import org.jsoup.nodes.Document; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; -import org.jsoup.nodes.Element; public class HitomiRipper extends AbstractHTMLRipper { @@ -47,9 +47,9 @@ public class HitomiRipper extends AbstractHTMLRipper { } @Override - public Document getFirstPage() throws IOException { + public Document getFirstPage() throws IOException, URISyntaxException { // if we go to /GALLERYID.js we get a nice json array of all images in the gallery - return Http.url(new URL(url.toExternalForm().replaceAll("hitomi", "ltn.hitomi").replaceAll(".html", ".js"))).ignoreContentType().get(); + return Http.url(new URI(url.toExternalForm().replaceAll("hitomi", "ltn.hitomi").replaceAll(".html", ".js")).toURL()).ignoreContentType().get(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java index 30da2344..7e26faa2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java @@ -8,6 +8,8 @@ import org.json.JSONObject; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -34,7 +36,7 @@ public class NsfwXxxRipper extends AbstractJSONRipper { @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { String u = url.toExternalForm(); // https://nsfw.xxx/user/kelly-kat/foo -> https://nsfw.xxx/user/kelly-kat // https://nsfw.xxx/user/kelly-kat -> https://nsfw.xxx/user/kelly-kat @@ -44,15 +46,15 @@ public class NsfwXxxRipper extends AbstractJSONRipper { throw new MalformedURLException("Invalid URL: " + url); } - return new URL(u); + return new URI(u).toURL(); } String getUser() throws MalformedURLException { return getGID(url); } - URL getPage(int page) throws MalformedURLException { - return new URL("https://nsfw.xxx/slide-page/" + page + "?nsfw%5B%5D=0&types%5B%5D=image&types%5B%5D=video&types%5B%5D=gallery&slider=1&jsload=1&user=" + getUser()); + URL getPage(int page) throws MalformedURLException, URISyntaxException { + return new URI("https://nsfw.xxx/slide-page/" + page + "?nsfw%5B%5D=0&types%5B%5D=image&types%5B%5D=video&types%5B%5D=gallery&slider=1&jsload=1&user=" + getUser()).toURL(); } @@ -71,18 +73,18 @@ public class NsfwXxxRipper extends AbstractJSONRipper { int currentPage = 1; @Override - protected JSONObject getFirstPage() throws IOException { + protected JSONObject getFirstPage() throws IOException, URISyntaxException { return Http.url(getPage(1)).getJSON(); } List descriptions = new ArrayList<>(); @Override - protected JSONObject getNextPage(JSONObject doc) throws IOException { + protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException { currentPage++; JSONObject nextPage = Http.url(getPage(doc.getInt("page") + 1)).getJSON(); JSONArray items = nextPage.getJSONArray("items"); - if (items.length() == 0) { + if (items.isEmpty()) { throw new IOException("No more pages"); } return nextPage; @@ -120,7 +122,7 @@ public class NsfwXxxRipper extends AbstractJSONRipper { return new ApiEntry(srcUrl, o.getString("author"), o.getString("title")); }) - .collect(Collectors.toList()); + .toList(); data.forEach(e -> descriptions.add(e.title)); return data.stream().map(e -> e.srcUrl).collect(Collectors.toList()); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 4c62a184..2cea95a7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -2,6 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -44,13 +45,13 @@ public class XhamsterRipper extends AbstractHTMLRipper { } @Override - public URL sanitizeURL(URL url) throws MalformedURLException { + public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { if (isVideoUrl(url)) { return url; } String URLToReturn = url.toExternalForm(); URLToReturn = URLToReturn.replaceAll("https?://\\w?\\w?\\.?xhamster([^<]*)\\.", "https://m.xhamster$1."); - URL san_url = new URL(URLToReturn); + URL san_url = new URI(URLToReturn).toURL(); LOGGER.info("sanitized URL is " + san_url.toExternalForm()); return san_url; } @@ -168,10 +169,10 @@ public class XhamsterRipper extends AbstractHTMLRipper { // This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to // the page chamster.com but displays the mobile site from m.xhamster.com pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster([^<]*)\\.", "://m.xhamster$1."); - String image = Http.url(new URL(pageWithImageUrl)).get().select("a > img#photoCurr").attr("src"); + String image = Http.url(new URI(pageWithImageUrl).toURL()).get().select("a > img#photoCurr").attr("src"); result.add(image); downloadFile(image); - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { LOGGER.error("Was unable to load page " + pageWithImageUrl); } if (isStopped() || isThisATest()) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java index 16526945..a9c39a9c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -54,7 +56,7 @@ public class CliphunterRipper extends VideoRipper { } @Override - public void rip() throws IOException { + public void rip() throws IOException, URISyntaxException { LOGGER.info("Retrieving " + this.url); String html = Http.url(url).get().html(); String jsonString = html.substring(html.indexOf("var flashVars = {d: '") + 21); @@ -71,7 +73,7 @@ public class CliphunterRipper extends VideoRipper { vidURL += c; } } - addURLToDownload(new URL(vidURL), HOST + "_" + getGID(this.url)); + addURLToDownload(new URI(vidURL).toURL(), HOST + "_" + getGID(this.url)); waitForThreads(); } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java index 678435af..5a3dcebb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; @@ -56,7 +58,7 @@ public class PornhubRipper extends VideoRipper { } @Override - public void rip() throws IOException { + public void rip() throws IOException, URISyntaxException { String vidUrl = ""; LOGGER.info(" Retrieving " + this.url.toExternalForm()); Document doc = Http.url(this.url).get(); @@ -146,7 +148,7 @@ public class PornhubRipper extends VideoRipper { if (vidUrl.equals("")) { throw new IOException("Unable to find encrypted video URL at " + this.url); } - addURLToDownload(new URL(vidUrl), HOST + "_" + bestQuality + "p_" + getGID(this.url)); + addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + bestQuality + "p_" + getGID(this.url)); waitForThreads(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/StickyXXXRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/StickyXXXRipper.java index 7c951b23..8708e552 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/StickyXXXRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/StickyXXXRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -32,11 +34,6 @@ public class StickyXXXRipper extends VideoRipper { return m.matches(); } - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return url; - } - @Override public String getGID(URL url) throws MalformedURLException { Pattern p = Pattern.compile("^https?://.*stickyxxx\\.com(/)(.*)/$"); @@ -52,7 +49,7 @@ public class StickyXXXRipper extends VideoRipper { } @Override - public void rip() throws IOException { + public void rip() throws IOException, URISyntaxException { LOGGER.info("Retrieving " + this.url); Document doc = Http.url(url).get(); Elements videos = doc.select(".wp-video > video > source"); @@ -60,7 +57,7 @@ public class StickyXXXRipper extends VideoRipper { throw new IOException("Could not find Embed code at " + url); } String vidUrl = videos.attr("src"); - addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url)); + addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); waitForThreads(); } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java index c72a5a59..bd4ee556 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -34,11 +36,6 @@ public class TwitchVideoRipper extends VideoRipper { return m.matches(); } - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return url; - } - @Override public String getGID(URL url) throws MalformedURLException { Pattern p = Pattern.compile("^https://clips\\.twitch\\.tv/(.*)$"); @@ -54,7 +51,7 @@ public class TwitchVideoRipper extends VideoRipper { } @Override - public void rip() throws IOException { + public void rip() throws IOException, URISyntaxException { LOGGER.info("Retrieving " + this.url); Document doc = Http.url(url).get(); @@ -72,7 +69,7 @@ public class TwitchVideoRipper extends VideoRipper { Matcher m = p.matcher(element.data()); if (m.find()){ String vidUrl = m.group(1); - addURLToDownload(new URL(vidUrl), HOST + "_" + title); + addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + title); } } waitForThreads(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java index 078b32a5..279e1d3a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -52,7 +54,7 @@ public class ViddmeRipper extends VideoRipper { } @Override - public void rip() throws IOException { + public void rip() throws IOException, URISyntaxException { LOGGER.info(" Retrieving " + this.url.toExternalForm()); Document doc = Http.url(this.url).get(); Elements videos = doc.select("meta[name=twitter:player:stream]"); @@ -61,7 +63,7 @@ public class ViddmeRipper extends VideoRipper { } String vidUrl = videos.first().attr("content"); vidUrl = vidUrl.replaceAll("&", "&"); - addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url)); + addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); waitForThreads(); } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java index 052b2cbe..3fbb6375 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.List; import java.util.regex.Matcher; @@ -33,11 +35,6 @@ public class VidearnRipper extends VideoRipper { return m.matches(); } - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - return url; - } - @Override public String getGID(URL url) throws MalformedURLException { Pattern p = Pattern.compile("^https?://[wm.]*videarn\\.com/[a-zA-Z0-9\\-]+/([0-9]+).*$"); @@ -53,15 +50,15 @@ public class VidearnRipper extends VideoRipper { } @Override - public void rip() throws IOException { + public void rip() throws IOException, URISyntaxException { LOGGER.info("Retrieving " + this.url); Document doc = Http.url(url).get(); List mp4s = Utils.between(doc.html(), "file:\"", "\""); if (mp4s.isEmpty()) { throw new IOException("Could not find files at " + url); } - String vidUrl = mp4s.get(0); - addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url)); + String vidUrl = mp4s.getFirst(); + addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); waitForThreads(); } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java index 70528727..84206abb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java @@ -2,6 +2,8 @@ package com.rarchives.ripme.ripper.rippers.video; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -51,10 +53,10 @@ public class VkRipper extends VideoRipper { } @Override - public void rip() throws IOException { + public void rip() throws IOException, URISyntaxException { LOGGER.info(" Retrieving " + this.url); String videoURL = getVideoURLAtPage(this.url.toExternalForm()); - addURLToDownload(new URL(videoURL), HOST + "_" + getGID(this.url)); + addURLToDownload(new URI(videoURL).toURL(), HOST + "_" + getGID(this.url)); waitForThreads(); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StickyXXXRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StickyXXXRipperTest.java index 57a07d90..71038c94 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StickyXXXRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/StickyXXXRipperTest.java @@ -7,9 +7,11 @@ import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.video.StickyXXXRipper; // import com.rarchives.ripme.tst.ripper.rippers.RippersTest; import com.rarchives.ripme.utils.Utils; +import org.junit.jupiter.api.Test; public class StickyXXXRipperTest extends RippersTest { + @Test public void testStickyXXXVideo() throws IOException, URISyntaxException { // This test fails on the CI - possibly due to checking for a file before it's written - so we're skipping it if (Utils.getConfigBoolean("test.run_flaky_tests", false)) { From edc5d9d5689d17bb34acd8bf14f038f75c3619a7 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 29 Mar 2024 23:19:59 +0100 Subject: [PATCH 467/512] update jacoco for java-22 --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 30f93110..e4c08af6 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -43,7 +43,7 @@ version = "1.7.94" description = "ripme" jacoco { - toolVersion = "0.8.10" + toolVersion = "0.8.11" } jgitver { From 6977025e0f05948b3d578e1602ee167142c69bbd Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 6 Apr 2024 14:55:36 +0200 Subject: [PATCH 468/512] abstractripper new uri instead url --- src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 481c694f..f290e72b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -8,6 +8,7 @@ import java.io.FileWriter; import java.io.IOException; import java.lang.reflect.Constructor; import java.net.MalformedURLException; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Files; @@ -302,8 +303,8 @@ public abstract class AbstractRipper if (url.toExternalForm().contains(" ")) { // If for some reason the url with all spaces encoded as %20 is malformed print an error try { - url = new URL(url.toExternalForm().replaceAll(" ", "%20")); - } catch (MalformedURLException e) { + url = new URI(url.toExternalForm().replaceAll(" ", "%20")).toURL(); + } catch (MalformedURLException | URISyntaxException e) { LOGGER.error("Unable to remove spaces from url\nURL: " + url.toExternalForm()); e.printStackTrace(); } From 7ea284174d8b5f6ccd48f95632ef554d3569dc56 Mon Sep 17 00:00:00 2001 From: joroto <47276635+joroto@users.noreply.github.com> Date: Thu, 20 Jun 2024 08:44:25 +0300 Subject: [PATCH 469/512] Imagebam ripper fixed (#188) --- .../rarchives/ripme/ripper/rippers/ImagebamRipper.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index 596680dd..0699273f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -10,11 +10,14 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang.StringUtils; +import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -128,7 +131,12 @@ public class ImagebamRipper extends AbstractHTMLRipper { */ private void fetchImage() { try { - Document doc = Http.url(url).get(); + Map cookies = new HashMap<>(); + cookies.put("nsfw_inter", "1"); + Document doc = Jsoup.connect(url.toString()) + .cookies(cookies) + .get(); + // Find image Elements metaTags = doc.getElementsByTag("meta"); From a178d8f6b914d3a443f56ef4a12a329e3a73ea57 Mon Sep 17 00:00:00 2001 From: xufeiranfree Date: Thu, 20 Jun 2024 13:47:26 +0800 Subject: [PATCH 470/512] Unify colons in UI (#189) --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 2 +- src/main/resources/LabelsBundle.properties | 2 +- src/main/resources/LabelsBundle_de_DE.properties | 4 ++-- src/main/resources/LabelsBundle_el_GR.properties | 2 +- src/main/resources/LabelsBundle_fi_FI.properties | 4 ++-- src/main/resources/LabelsBundle_fr_CH.properties | 4 ++-- src/main/resources/LabelsBundle_in_ID.properties | 4 ++-- src/main/resources/LabelsBundle_it_IT.properties | 2 +- src/main/resources/LabelsBundle_kr_KR.properties | 4 ++-- src/main/resources/LabelsBundle_nl_NL.properties | 4 ++-- src/main/resources/LabelsBundle_pl_PL.properties | 2 +- src/main/resources/LabelsBundle_porrisavvo_FI.properties | 4 ++-- src/main/resources/LabelsBundle_pt_BR.properties | 2 +- src/main/resources/LabelsBundle_pt_PT.properties | 4 ++-- src/main/resources/LabelsBundle_ru_RU.properties | 2 +- src/main/resources/LabelsBundle_zh_CN.properties | 2 +- 16 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index e5cc2f88..8b547d7d 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -535,7 +535,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { configUpdateButton = new JButton(Utils.getLocalizedString("check.for.updates")); configUpdateLabel = new JLabel( Utils.getLocalizedString("current.version") + ": " + UpdateUtils.getThisJarVersion(), JLabel.RIGHT); - configThreadsLabel = new JLabel(Utils.getLocalizedString("max.download.threads") + ":", JLabel.RIGHT); + configThreadsLabel = new JLabel(Utils.getLocalizedString("max.download.threads"), JLabel.RIGHT); configTimeoutLabel = new JLabel(Utils.getLocalizedString("timeout.mill"), JLabel.RIGHT); configRetriesLabel = new JLabel(Utils.getLocalizedString("retry.download.count"), JLabel.RIGHT); configRetrySleepLabel = new JLabel(Utils.getLocalizedString("retry.sleep.mill"), JLabel.RIGHT); diff --git a/src/main/resources/LabelsBundle.properties b/src/main/resources/LabelsBundle.properties index 63bca351..983086c2 100644 --- a/src/main/resources/LabelsBundle.properties +++ b/src/main/resources/LabelsBundle.properties @@ -12,7 +12,7 @@ check.for.updates = Check for updates auto.update = Auto-update? max.download.threads = Maximum download threads: timeout.mill = Timeout (in milliseconds): -retry.download.count = Retry download count +retry.download.count = Retry download count: retry.sleep.mill = Wait between retries (in milliseconds): overwrite.existing.files = Overwrite existing files? sound.when.rip.completes = Sound when rip completes diff --git a/src/main/resources/LabelsBundle_de_DE.properties b/src/main/resources/LabelsBundle_de_DE.properties index 9e767566..b13a0bb9 100644 --- a/src/main/resources/LabelsBundle_de_DE.properties +++ b/src/main/resources/LabelsBundle_de_DE.properties @@ -10,9 +10,9 @@ Configuration = Konfiguration current.version = Aktuelle Version check.for.updates = Suche nach Aktualisierungen auto.update = Automatisch Aktualisieren? -max.download.threads = Maximale Download-Threads +max.download.threads = Maximale Download-Threads: timeout.mill = Timeout (in Milliseconds): -retry.download.count = Anzahl der Downloadversuche +retry.download.count = Anzahl der Downloadversuche: overwrite.existing.files = Überschreibe bereits existierende Dateien? sound.when.rip.completes = Ton abspielen bei fertigem Download preserve.order = Reihenfolge beibehalten diff --git a/src/main/resources/LabelsBundle_el_GR.properties b/src/main/resources/LabelsBundle_el_GR.properties index 573e76e6..14656e87 100644 --- a/src/main/resources/LabelsBundle_el_GR.properties +++ b/src/main/resources/LabelsBundle_el_GR.properties @@ -12,7 +12,7 @@ check.for.updates = Έλεγχος για ενημερώσεις auto.update = Αυτόματη ενημέρωση? max.download.threads = Μέγιστος αριθμός παράλληλων συνδέσεων: timeout.mill = Λήξη (σε χιλιοστά του δευτερολέπτου): -retry.download.count = Αριθμός επανάληψεων μεταφόρτωσης +retry.download.count = Αριθμός επανάληψεων μεταφόρτωσης: overwrite.existing.files = Να αντικατασταθούν τα υπάρχοντα αρχεία? sound.when.rip.completes = Ήχος όταν το rip ολοκληρωθεί preserve.order = Διατήρηση σειράς diff --git a/src/main/resources/LabelsBundle_fi_FI.properties b/src/main/resources/LabelsBundle_fi_FI.properties index 6edd4e45..c823cf05 100644 --- a/src/main/resources/LabelsBundle_fi_FI.properties +++ b/src/main/resources/LabelsBundle_fi_FI.properties @@ -10,9 +10,9 @@ Configuration = Asetukset current.version = Nykyinen versio check.for.updates = Tarkista päivitykset auto.update = Automaattipäivitys? -max.download.threads = Yhtäaikaiset lataukset +max.download.threads = Yhtäaikaiset lataukset: timeout.mill = Aikakatkaisu (millisekunneissa): -retry.download.count = Latauksen uudelleenyritykset +retry.download.count = Latauksen uudelleenyritykset: overwrite.existing.files = Korvaa nykyiset tiedostot? sound.when.rip.completes = Valmistumisääni preserve.order = Pidä järjestys diff --git a/src/main/resources/LabelsBundle_fr_CH.properties b/src/main/resources/LabelsBundle_fr_CH.properties index b489e3e3..1b035dac 100644 --- a/src/main/resources/LabelsBundle_fr_CH.properties +++ b/src/main/resources/LabelsBundle_fr_CH.properties @@ -10,9 +10,9 @@ Configuration = Configuration current.version = Version actuelle check.for.updates = Vérifier mises à jour auto.update = Mises à jour automatiques? -max.download.threads = Nombre de téléchargements parallèles maximum +max.download.threads = Nombre de téléchargements parallèles maximum: timeout.mill = Délai d'expiration (en millisecondes): -retry.download.count = Nombre d'essais téléchargement +retry.download.count = Nombre d'essais téléchargement: overwrite.existing.files = Remplacer fichiers existants ? sound.when.rip.completes = Son lorsque le rip est terminé preserve.order = Conserver l'ordre diff --git a/src/main/resources/LabelsBundle_in_ID.properties b/src/main/resources/LabelsBundle_in_ID.properties index b5e773d5..778e72ab 100644 --- a/src/main/resources/LabelsBundle_in_ID.properties +++ b/src/main/resources/LabelsBundle_in_ID.properties @@ -10,9 +10,9 @@ Configuration = Pengaturan current.version = Versi saat ini check.for.updates = Periksa update auto.update = Update otomatis? -max.download.threads = Thread unduh maksimal +max.download.threads = Thread unduh maksimal: timeout.mill = Batas waktu (dalam milidetik): -retry.download.count = Jumlah percobaan unduh +retry.download.count = Jumlah percobaan unduh: overwrite.existing.files = Timpa file yang ada? sound.when.rip.completes = Hidupkan suara saat rip selesai preserve.order = Pertahankan urutan diff --git a/src/main/resources/LabelsBundle_it_IT.properties b/src/main/resources/LabelsBundle_it_IT.properties index de00612b..192d777a 100644 --- a/src/main/resources/LabelsBundle_it_IT.properties +++ b/src/main/resources/LabelsBundle_it_IT.properties @@ -12,7 +12,7 @@ check.for.updates = Controlla aggiornamenti auto.update = Aggiornamento automatico? max.download.threads = Thread di download massimi: timeout.mill = Timeout (in millisecondi): -retry.download.count = Tentativi di download +retry.download.count = Tentativi di download: overwrite.existing.files = Sovrascrivi file esistenti? sound.when.rip.completes = Suono al completamento del rip preserve.order = Preserva ordine diff --git a/src/main/resources/LabelsBundle_kr_KR.properties b/src/main/resources/LabelsBundle_kr_KR.properties index e0110055..984da15b 100644 --- a/src/main/resources/LabelsBundle_kr_KR.properties +++ b/src/main/resources/LabelsBundle_kr_KR.properties @@ -10,9 +10,9 @@ Configuration = \uAD6C\uC131 current.version = \uD604\uC7AC \uBC84\uC804 check.for.updates = \uC5C5\uB370\uC774\uD2B8 \uD655\uC778 auto.update = \uC790\uB3D9 \uC5C5\uB370\uC774\uD2B8 -max.download.threads = \uCD5C\uB300 \uB2E4\uC6B4\uB85C\uB4DC \uC4F0\uB808\uB4DC \uC218 +max.download.threads = \uCD5C\uB300 \uB2E4\uC6B4\uB85C\uB4DC \uC4F0\uB808\uB4DC \uC218: timeout.mill = \uC2DC\uAC04 \uC81C\uD55C (\uBC00\uB9AC\uCD08): -retry.download.count = \uB2E4\uC6B4\uB85C\uB4DC \uC7AC\uC2DC\uB3C4 \uD68C\uC218 +retry.download.count = \uB2E4\uC6B4\uB85C\uB4DC \uC7AC\uC2DC\uB3C4 \uD68C\uC218: overwrite.existing.files = \uC911\uBCF5\uD30C\uC77C \uB36E\uC5B4\uC4F0\uAE30 sound.when.rip.completes = \uC644\uB8CC\uC2DC \uC54C\uB9BC preserve.order = \uBA85\uB839 \uAE30\uC5B5\uD558\uAE30 diff --git a/src/main/resources/LabelsBundle_nl_NL.properties b/src/main/resources/LabelsBundle_nl_NL.properties index e1d9d61c..6cec1f73 100644 --- a/src/main/resources/LabelsBundle_nl_NL.properties +++ b/src/main/resources/LabelsBundle_nl_NL.properties @@ -10,9 +10,9 @@ Configuration = Configuratie current.version = Huidige versie check.for.updates = Controleer op updates auto.update = Auto-update? -max.download.threads = Maximale downloadthreads +max.download.threads = Maximale downloadthreads: timeout.mill = Timeout (in milliseconden): -retry.download.count = Aantal keren opnieuw proberen te downloaden +retry.download.count = Aantal keren opnieuw proberen te downloaden: overwrite.existing.files = Bestaande bestanden overschrijven? sound.when.rip.completes = Geluid wanneer rip klaar is preserve.order = Volgorde behouden diff --git a/src/main/resources/LabelsBundle_pl_PL.properties b/src/main/resources/LabelsBundle_pl_PL.properties index dbb74ef1..a3bbbb38 100644 --- a/src/main/resources/LabelsBundle_pl_PL.properties +++ b/src/main/resources/LabelsBundle_pl_PL.properties @@ -12,7 +12,7 @@ check.for.updates = Sprawdź dostępność aktualizacji auto.update = Auto Aktualizacja? max.download.threads = Maksymalna Ilośc Pobieranych Plików: timeout.mill = Opóźnienie (w milisekundach): -retry.download.count = Liczba ponownych pobrań +retry.download.count = Liczba ponownych pobrań: overwrite.existing.files = Nadpisać istniejące pliki? sound.when.rip.completes = Dźwięk po zakończeniu preserve.order = Zachować porządek diff --git a/src/main/resources/LabelsBundle_porrisavvo_FI.properties b/src/main/resources/LabelsBundle_porrisavvo_FI.properties index a2ba056e..653709ab 100644 --- a/src/main/resources/LabelsBundle_porrisavvo_FI.properties +++ b/src/main/resources/LabelsBundle_porrisavvo_FI.properties @@ -10,9 +10,9 @@ Configuration = Assetuksse current.version = Nykyne versijjo check.for.updates = Tarkist update auto.update = Automaatpäivvitys? -max.download.threads = Yht'aikasse ripi +max.download.threads = Yht'aikasse ripi: timeout.mill = Timeout (millisekois): -retry.download.count = Ripi retry count +retry.download.count = Ripi retry count: overwrite.existing.files = Korvvaa nykysse filu? sound.when.rip.completes = Valmistummis'ään preserve.order = Pir järestys diff --git a/src/main/resources/LabelsBundle_pt_BR.properties b/src/main/resources/LabelsBundle_pt_BR.properties index 88209235..91c262b7 100644 --- a/src/main/resources/LabelsBundle_pt_BR.properties +++ b/src/main/resources/LabelsBundle_pt_BR.properties @@ -12,7 +12,7 @@ check.for.updates = Verificar atualizações auto.update = Atualização automática? max.download.threads = Número máximo de conexões: timeout.mill = Tempo limite (em milissegundos): -retry.download.count = Número de tentativas +retry.download.count = Número de tentativas: overwrite.existing.files = Sobrescrever arquivos existentes? sound.when.rip.completes = Som quando terminar o rip preserve.order = Preservar ordem diff --git a/src/main/resources/LabelsBundle_pt_PT.properties b/src/main/resources/LabelsBundle_pt_PT.properties index a0058524..500049ce 100644 --- a/src/main/resources/LabelsBundle_pt_PT.properties +++ b/src/main/resources/LabelsBundle_pt_PT.properties @@ -10,9 +10,9 @@ open = Abrir current.version = Versão atual check.for.updates = Verificar atualizações auto.update = Atualização automática? -max.download.threads = Número máximo de processos de transferência +max.download.threads = Número máximo de processos de transferência: timeout.mill = Tempo de espera (em milissegundos): -retry.download.count = Número de novas tentativas de transferência +retry.download.count = Número de novas tentativas de transferência: overwrite.existing.files = Sobrescrever ficheiros existentes? sound.when.rip.completes = Notificar quando o rip é concluído preserve.order = Manter a ordem diff --git a/src/main/resources/LabelsBundle_ru_RU.properties b/src/main/resources/LabelsBundle_ru_RU.properties index a3100df8..f354d15a 100644 --- a/src/main/resources/LabelsBundle_ru_RU.properties +++ b/src/main/resources/LabelsBundle_ru_RU.properties @@ -12,7 +12,7 @@ check.for.updates = Проверить обновления auto.update = Автообновление max.download.threads = Максимальное число потоков: timeout.mill = Задержка (в миллисекундах): -retry.download.count = Число повторов +retry.download.count = Число повторов: overwrite.existing.files = Перезаписать существующие файлы? sound.when.rip.completes = Звук при завершении preserve.order = Сохранять порядок diff --git a/src/main/resources/LabelsBundle_zh_CN.properties b/src/main/resources/LabelsBundle_zh_CN.properties index cd43da1c..7cf6d781 100644 --- a/src/main/resources/LabelsBundle_zh_CN.properties +++ b/src/main/resources/LabelsBundle_zh_CN.properties @@ -12,7 +12,7 @@ check.for.updates = 检查更新 auto.update = 自动更新？ max.download.threads = 最大下载线程数： timeout.mill = 超时（毫秒）： -retry.download.count = 重试下载次数 +retry.download.count = 重试下载次数： overwrite.existing.files = 覆盖现有文件？ sound.when.rip.completes = 抓取完成时播放声音 preserve.order = 保持顺序 From 71e20fe851c4c0201989b1321b72a8f4fc5b7ccb Mon Sep 17 00:00:00 2001 From: brant spar Date: Fri, 28 Jun 2024 13:09:31 +1000 Subject: [PATCH 471/512] - Fixed Motherless ripper - increased default retries to 3 (why only 1?) - fixed deprecation message for Motherless ripper --- .../ripper/rippers/MotherlessRipper.java | 19 ++++++++++++++----- .../java/com/rarchives/ripme/utils/Http.java | 2 +- src/main/resources/rip.properties | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java index ee657fb9..4269ff2e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java @@ -5,6 +5,8 @@ import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; @@ -61,7 +63,7 @@ public class MotherlessRipper extends AbstractHTMLRipper { if (!notHome) { StringBuilder newPath = new StringBuilder(path); newPath.insert(2, "M"); - firstURL = new URL(this.url, "https://" + DOMAIN + newPath); + firstURL = URI.create("https://" + DOMAIN + newPath).toURL(); LOGGER.info("Changed URL to " + firstURL); } return Http.url(firstURL).referrer("https://motherless.com").get(); @@ -69,6 +71,9 @@ public class MotherlessRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException, URISyntaxException { + + Files.write(Paths.get("doc-next-page.txt"), doc.outerHtml().getBytes()); + Elements nextPageLink = doc.head().select("link[rel=next]"); if (nextPageLink.isEmpty()) { throw new IOException("Last page reached"); @@ -111,7 +116,7 @@ public class MotherlessRipper extends AbstractHTMLRipper { @Override protected void downloadURL(URL url, int index) { // Create thread for finding image at "url" page - MotherlessImageThread mit = new MotherlessImageThread(url, index); + MotherlessImageRunnable mit = new MotherlessImageRunnable(url, index); motherlessThreadPool.addThread(mit); try { Thread.sleep(IMAGE_SLEEP_TIME); @@ -150,15 +155,19 @@ public class MotherlessRipper extends AbstractHTMLRipper { throw new MalformedURLException("Expected URL format: https://motherless.com/GIXXXXXXX, got: " + url); } - + @Override + protected DownloadThreadPool getThreadPool() { + return motherlessThreadPool; + } + /** * Helper class to find and download images found on "image" pages */ - private class MotherlessImageThread implements Runnable { + private class MotherlessImageRunnable implements Runnable { private final URL url; private final int index; - MotherlessImageThread(URL url, int index) { + MotherlessImageRunnable(URL url, int index) { super(); this.url = url; this.index = index; diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index f3b163a0..374f32e7 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -56,7 +56,7 @@ public class Http { } private void defaultSettings() { - this.retries = Utils.getConfigInteger("download.retries", 1); + this.retries = Utils.getConfigInteger("download.retries", 3); this.retrySleep = Utils.getConfigInteger("download.retry.sleep", 5000); connection = Jsoup.connect(this.url); connection.userAgent(AbstractRipper.USER_AGENT); diff --git a/src/main/resources/rip.properties b/src/main/resources/rip.properties index 35d6c123..484cacac 100644 --- a/src/main/resources/rip.properties +++ b/src/main/resources/rip.properties @@ -6,7 +6,7 @@ threads.size = 5 file.overwrite = false # Number of retries on failed downloads -download.retries = 1 +download.retries = 3 # File download timeout (in milliseconds) download.timeout = 60000 From e938ac97b1e39def21a95f89185b103bb3c51173 Mon Sep 17 00:00:00 2001 From: brant spar Date: Tue, 2 Jul 2024 15:10:44 +1000 Subject: [PATCH 472/512] fixed the vidble test URL --- .../rarchives/ripme/ripper/rippers/MotherlessRipper.java | 5 ----- .../ripme/tst/ripper/rippers/VidbleRipperTest.java | 6 +++--- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java index 4269ff2e..d2af02a1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java @@ -5,8 +5,6 @@ import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; @@ -17,7 +15,6 @@ import org.jsoup.nodes.Element; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.DownloadThreadPool; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; import org.jsoup.select.Elements; @@ -72,8 +69,6 @@ public class MotherlessRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException, URISyntaxException { - Files.write(Paths.get("doc-next-page.txt"), doc.outerHtml().getBytes()); - Elements nextPageLink = doc.head().select("link[rel=next]"); if (nextPageLink.isEmpty()) { throw new IOException("Last page reached"); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java index 7e3799f9..0ce64540 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/VidbleRipperTest.java @@ -12,15 +12,15 @@ import org.junit.jupiter.api.Test; public class VidbleRipperTest extends RippersTest { @Test public void testVidbleRip() throws IOException, URISyntaxException { - VidbleRipper ripper = new VidbleRipper(new URI("http://www.vidble.com/album/y1oyh3zd").toURL()); + VidbleRipper ripper = new VidbleRipper(new URI("https://vidble.com/album/cGEFr8zi").toURL()); testRipper(ripper); } @Test public void testGetGID() throws IOException, URISyntaxException { - URL url = new URI("http://www.vidble.com/album/y1oyh3zd").toURL(); + URL url = new URI("https://vidble.com/album/cGEFr8zi").toURL(); VidbleRipper ripper = new VidbleRipper(url); - Assertions.assertEquals("y1oyh3zd", ripper.getGID(url)); + Assertions.assertEquals("cGEFr8zi", ripper.getGID(url)); } } From 4e595b143b61d6b70a39402d3b2da15bfb2f4c83 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sat, 27 Jul 2024 11:09:43 +0200 Subject: [PATCH 473/512] java locale painful, fix - and _ java uses ietf-bcp-47 lanauge tags, which are strings separated by -. resource bundle names are _ separated. porrisavo not fixed, this is a one person artificial dialect from finish cities of pori and savo. --- src/main/java/com/rarchives/ripme/utils/Utils.java | 11 ++++++----- ...erties => LabelsBundle_fi_FI_porrisavo.properties} | 0 2 files changed, 6 insertions(+), 5 deletions(-) rename src/main/resources/{LabelsBundle_porrisavvo_FI.properties => LabelsBundle_fi_FI_porrisavo.properties} (100%) diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index a1f85d11..88eb0c5e 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -53,8 +53,6 @@ import java.util.regex.Pattern; */ public class Utils { - private static final Pattern pattern = Pattern.compile("LabelsBundle_(?[A-Za-z_]+).properties"); - private static final String DEFAULT_LANG = "en_US"; private static final String RIP_DIRECTORY = "rips"; private static final String CONFIG_FILE = "rip.properties"; private static final String OS = System.getProperty("os.name").toLowerCase(); @@ -740,8 +738,8 @@ public class Utils { new UTF8Control()); } } else { - String[] langCode = langSelect.split("_"); - LOGGER.info("Setting locale to " + langSelect); + String[] langCode = langSelect.split("-"); + LOGGER.info("set locale, langcoe: {}, selected langauge: {}, locale: {}", langCode, langSelect, Locale.forLanguageTag(langSelect)); return ResourceBundle.getBundle("LabelsBundle", Locale.forLanguageTag(langSelect), new UTF8Control()); } try { @@ -755,6 +753,7 @@ public class Utils { public static void setLanguage(String langSelect) { resourceBundle = getResourceBundle(langSelect); + LOGGER.info("Selected resource bundle locale: {}, from {}", resourceBundle.getLocale().toString(), langSelect); } public static String getSelectedLanguage() { @@ -763,6 +762,8 @@ public class Utils { // All the langs ripme has been translated into public static String[] getSupportedLanguages() { + final Pattern pattern = Pattern.compile("LabelsBundle_(?[A-Za-z_]+).properties"); + final String DEFAULT_LANG = "en-US"; ArrayList filesList = new ArrayList<>(); try { URI uri = Objects.requireNonNull(Utils.class.getResource("/rip.properties")).toURI(); @@ -782,7 +783,7 @@ public class Utils { for (int i = 0; i < filesList.size(); i++) { Matcher matcher = pattern.matcher(filesList.get(i).toString()); if (matcher.find()) - langs[i] = matcher.group("lang"); + langs[i] = matcher.group("lang").replace("_", "-"); } return langs; diff --git a/src/main/resources/LabelsBundle_porrisavvo_FI.properties b/src/main/resources/LabelsBundle_fi_FI_porrisavo.properties similarity index 100% rename from src/main/resources/LabelsBundle_porrisavvo_FI.properties rename to src/main/resources/LabelsBundle_fi_FI_porrisavo.properties From aa00c6f612b5e54138113e61e7264c8b038f06bf Mon Sep 17 00:00:00 2001 From: joroto Date: Mon, 29 Jul 2024 19:17:27 +0300 Subject: [PATCH 474/512] QueueMenuMouseListener right click menu fix --- .../rarchives/ripme/ui/QueueMenuMouseListener.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java b/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java index 08adce80..0be4b46f 100644 --- a/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java +++ b/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java @@ -58,9 +58,18 @@ class QueueMenuMouseListener extends MouseAdapter { updateQueue.accept(queueListModel); } - @SuppressWarnings("unchecked") @Override - public void mouseClicked(MouseEvent e) { + public void mousePressed(MouseEvent e) { + checkPopupTrigger(e); + } + + @Override + public void mouseReleased(MouseEvent e) { + checkPopupTrigger(e); + } + + @SuppressWarnings("unchecked") + private void checkPopupTrigger(MouseEvent e) { if (e.getModifiersEx() == InputEvent.BUTTON3_DOWN_MASK) { if (!(e.getSource() instanceof JList)) { return; From 488849c2532b58481b64155c2f039fffa4d55815 Mon Sep 17 00:00:00 2001 From: Tush-r Date: Thu, 29 Aug 2024 17:21:01 +0530 Subject: [PATCH 475/512] Fix rgif ripper --- .../ripme/ripper/rippers/RedgifsRipper.java | 109 ++++++++++++------ 1 file changed, 74 insertions(+), 35 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java index 472d6d3a..81607ec5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java @@ -1,10 +1,8 @@ package com.rarchives.ripme.ripper.rippers; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; import org.json.JSONArray; import org.json.JSONObject; -import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -19,14 +17,26 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -public class RedgifsRipper extends AbstractHTMLRipper { +import org.apache.http.client.utils.URIBuilder; + +import com.rarchives.ripme.ripper.AbstractJSONRipper; + +public class RedgifsRipper extends AbstractJSONRipper { private static final String HOST = "redgifs.com"; private static final String HOST_2 = "gifdeliverynetwork.com"; + private static final String GIFS_DETAIL_ENDPOINT = "https://api.redgifs.com/v2/gifs/%s"; + private static final String USERS_SEARCH_ENDPOINT = "https://api.redgifs.com/v2/users/%s/search"; + private static final String TEMPORARY_AUTH_ENDPOINT = "https://api.redgifs.com/v2/auth/temporary"; String username = ""; + String authToken = ""; + // TODO remove String cursor = ""; - String count = "100"; + int count = 40; + int currentPage = 1; + int maxPages = 1; + // TODO remove with search String searchText = ""; int searchCount = 150; int searchStart = 0; @@ -73,18 +83,29 @@ public class RedgifsRipper extends AbstractHTMLRipper { } @Override - public Document getFirstPage() throws IOException { + public JSONObject getFirstPage() throws IOException { try { - if (!isProfile().matches() && !isSearch().matches()) { - return Http.url(url).get(); + if (authToken == null || authToken.equals("")){ + fetchAuthToken(); + } + + if (isSingleton().matches()) { + maxPages = 1; + String gifDetailsURL = String.format(GIFS_DETAIL_ENDPOINT, getGID(url)); + return Http.url(gifDetailsURL).header("Authorization", "Bearer " + authToken).getJSON(); } else if (isSearch().matches()) { - searchText = getGID(url).replace("-", " "); - return Http.url( - new URI("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart * searchCount).toURL()).ignoreContentType().get(); + // TODO fix search + // TODO remove + throw new IOException("TODO remove"); } else { username = getGID(url); - return Http.url(new URI("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count).toURL()) - .ignoreContentType().get(); + var uri = new URIBuilder(String.format(USERS_SEARCH_ENDPOINT, username)); + uri.addParameter("order", "new"); + uri.addParameter("count", Integer.toString(count)); + uri.addParameter("page", Integer.toString(currentPage)); + var json = Http.url(uri.build().toURL()).header("Authorization", "Bearer " + authToken).getJSON(); + maxPages = json.getInt("pages"); + return json; } } catch (URISyntaxException e) { throw new IOException(e); @@ -118,6 +139,7 @@ public class RedgifsRipper extends AbstractHTMLRipper { + " Got: " + url); } + // TODO remove private String stripHTMLTags(String t) { t = t.replaceAll("\n" + " \n" + @@ -130,42 +152,47 @@ public class RedgifsRipper extends AbstractHTMLRipper { } @Override - public Document getNextPage(Document doc) throws IOException, URISyntaxException { + public JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException { + if (currentPage == maxPages || isSingleton().matches()){ + return null; + } + currentPage++; if (isSearch().matches()) { - Document d = Http.url( - new URI("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText - + "&count=" + searchCount + "&start=" + searchCount*++searchStart).toURL()) - .ignoreContentType().get(); - return (hasURLs(d).isEmpty()) ? null : d; + // TODO search + // TODO remove + throw new IOException("// TODO remove"); + } else if (isProfile().matches()) { + var uri = new URIBuilder(String.format(USERS_SEARCH_ENDPOINT, getGID(url))); + uri.addParameter("order", "new"); + uri.addParameter("count", Integer.toString(count)); + uri.addParameter("page", Integer.toString(currentPage)); + var json = Http.url(uri.build().toURL()).header("Authorization", "Bearer " + authToken).getJSON(); + // Handle rare maxPages change during a rip + maxPages = json.getInt("pages"); + return json; } else { - if (cursor.equals("") || cursor.equals("null")) { - return null; - } else { - Document d = Http.url(new URI("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor).toURL()).ignoreContentType().get(); - return (hasURLs(d).isEmpty()) ? null : d; - } + return null; } } @Override - public List getURLsFromPage(Document doc) { + public List getURLsFromJSON(JSONObject json) { List result = new ArrayList<>(); if (isProfile().matches() || isSearch().matches()) { - result = hasURLs(doc); - } else { - Elements videos = doc.select("script"); - for (Element el : videos) { - String json = el.html(); - if (json.startsWith("{")) { - JSONObject page = new JSONObject(json); - result.add(page.getJSONObject("video").getString("contentUrl") - .replace("-mobile", "")); - } + // TODO check json keys for search + var gifs = json.getJSONArray("gifs"); + for (var gif : gifs){ + var hdURL = ((JSONObject)gif).getJSONObject("urls").getString("hd"); + result.add(hdURL); } + } else { + String hdURL = json.getJSONObject("gif").getJSONObject("urls").getString("hd"); + result.add(hdURL); } return result; } + // TODO delete /** * Helper method for retrieving URLs. * @param doc Document of the URL page to look through @@ -182,6 +209,7 @@ public class RedgifsRipper extends AbstractHTMLRipper { return result; } + // TODO delete /** * Helper method for retrieving video URLs. * @param url URL to gfycat page @@ -206,4 +234,15 @@ public class RedgifsRipper extends AbstractHTMLRipper { } throw new IOException(); } + + + /** + * Fetch a temorary auth token for the rip + * @throws IOException + */ + private void fetchAuthToken() throws IOException{ + var json = Http.url(TEMPORARY_AUTH_ENDPOINT).getJSON(); + var token = json.getString("token"); + authToken = token; + } } From 37e98846bb883d59af63e4a0f71a2cf5576d4864 Mon Sep 17 00:00:00 2001 From: Tush-r Date: Thu, 29 Aug 2024 17:21:01 +0530 Subject: [PATCH 476/512] Add support for rgif multiple image gallery --- .../ripme/ripper/rippers/RedgifsRipper.java | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java index 81607ec5..6a0f6ae6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java @@ -7,6 +7,8 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import static com.rarchives.ripme.App.logger; + import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; @@ -27,6 +29,7 @@ public class RedgifsRipper extends AbstractJSONRipper { private static final String HOST_2 = "gifdeliverynetwork.com"; private static final String GIFS_DETAIL_ENDPOINT = "https://api.redgifs.com/v2/gifs/%s"; private static final String USERS_SEARCH_ENDPOINT = "https://api.redgifs.com/v2/users/%s/search"; + private static final String GALLERY_ENDPOINT = "https://api.redgifs.com/v2/gallery/%s"; private static final String TEMPORARY_AUTH_ENDPOINT = "https://api.redgifs.com/v2/auth/temporary"; String username = ""; String authToken = ""; @@ -181,17 +184,54 @@ public class RedgifsRipper extends AbstractJSONRipper { if (isProfile().matches() || isSearch().matches()) { // TODO check json keys for search var gifs = json.getJSONArray("gifs"); - for (var gif : gifs){ + for (var gif : gifs) { + if (((JSONObject)gif).isNull("gallery")) { var hdURL = ((JSONObject)gif).getJSONObject("urls").getString("hd"); result.add(hdURL); + } else { + var galleryID = ((JSONObject)gif).getString("gallery"); + var gifID = ((JSONObject)gif).getString("id"); + result.addAll(getURLsForGallery(galleryID, gifID)); + } } } else { - String hdURL = json.getJSONObject("gif").getJSONObject("urls").getString("hd"); + var gif = json.getJSONObject("gif"); + if (gif.isNull("gallery")) { + String hdURL = gif.getJSONObject("urls").getString("hd"); result.add(hdURL); + } else { + var galleryID = gif.getString("gallery"); + var gifID = gif.getString("id"); + result.addAll(getURLsForGallery(galleryID, gifID)); + } } return result; } + + /** + * Get all images for a gif url with multiple images + * @param galleryID gallery id + * @param gifID gif id with multiple images for logging + * @return List + */ + private List getURLsForGallery(String galleryID, String gifID) { + List list = new ArrayList<>(); + if (galleryID == null || galleryID.isBlank()) { + return list; + } + try { + var json = Http.url(String.format(GALLERY_ENDPOINT, galleryID)).header("Authorization", "Bearer " + authToken).getJSON(); + for (var gif : json.getJSONArray("gifs")) { + var hdURL = ((JSONObject)gif).getJSONObject("urls").getString("hd"); + list.add(hdURL); + } + } catch (IOException e) { + logger.error(String.format("Error fetching gallery %s for gif %s", galleryID, gifID), e); + } + return list; + } + // TODO delete /** * Helper method for retrieving URLs. From 0589c52e116a2ac45c340eb202eec2d8aa7d8a75 Mon Sep 17 00:00:00 2001 From: Tush-r Date: Thu, 29 Aug 2024 17:21:01 +0530 Subject: [PATCH 477/512] Fix rgif search and add support for tag urls --- .../ripme/ripper/rippers/RedgifsRipper.java | 155 +++++++++++++++--- 1 file changed, 136 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java index 6a0f6ae6..99a5c230 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java @@ -1,6 +1,7 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.utils.Http; + import org.json.JSONArray; import org.json.JSONObject; import org.jsoup.nodes.Document; @@ -14,8 +15,14 @@ import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -30,6 +37,8 @@ public class RedgifsRipper extends AbstractJSONRipper { private static final String GIFS_DETAIL_ENDPOINT = "https://api.redgifs.com/v2/gifs/%s"; private static final String USERS_SEARCH_ENDPOINT = "https://api.redgifs.com/v2/users/%s/search"; private static final String GALLERY_ENDPOINT = "https://api.redgifs.com/v2/gallery/%s"; + private static final String SEARCH_ENDPOINT = "https://api.redgifs.com/v2/search/%s"; + private static final String TAGS_ENDPOINT = "https://api.redgifs.com/v2/gifs/search"; private static final String TEMPORARY_AUTH_ENDPOINT = "https://api.redgifs.com/v2/auth/temporary"; String username = ""; String authToken = ""; @@ -76,7 +85,12 @@ public class RedgifsRipper extends AbstractJSONRipper { } public Matcher isSearch() { - Pattern p = Pattern.compile("^https?://[wm.]*redgifs\\.com/gifs/browse/([a-zA-Z0-9_.-]+).*$"); + Pattern p = Pattern.compile("^https?:\\/\\/[wm.]*redgifs\\.com\\/search(?:\\/[a-zA-Z]+)?\\?.*?query=([a-zA-Z0-9-_+%]+).*$"); + return p.matcher(url.toExternalForm()); + } + + public Matcher isTags() { + Pattern p = Pattern.compile("^https?:\\/\\/[wm.]*redgifs\\.com\\/gifs\\/([a-zA-Z0-9_.,-]+).*$"); return p.matcher(url.toExternalForm()); } @@ -96,10 +110,10 @@ public class RedgifsRipper extends AbstractJSONRipper { maxPages = 1; String gifDetailsURL = String.format(GIFS_DETAIL_ENDPOINT, getGID(url)); return Http.url(gifDetailsURL).header("Authorization", "Bearer " + authToken).getJSON(); - } else if (isSearch().matches()) { - // TODO fix search - // TODO remove - throw new IOException("TODO remove"); + } else if (isSearch().matches() || isTags().matches()) { + var json = Http.url(getSearchOrTagsURL()).header("Authorization", "Bearer " + authToken).getJSON(); + maxPages = json.getInt("pages"); + return json; } else { username = getGID(url); var uri = new URIBuilder(String.format(USERS_SEARCH_ENDPOINT, username)); @@ -111,7 +125,7 @@ public class RedgifsRipper extends AbstractJSONRipper { return json; } } catch (URISyntaxException e) { - throw new IOException(e); + throw new IOException("Failed to build first page url", e); } } @@ -122,14 +136,35 @@ public class RedgifsRipper extends AbstractJSONRipper { @Override public String getGID(URL url) throws MalformedURLException { - Matcher m = isProfile(); if (m.matches()) { return m.group(1); } m = isSearch(); if (m.matches()) { - return m.group(1); + var sText = m.group(1); + if (sText == null || sText.isBlank()){ + throw new MalformedURLException(String.format("Expected redgifs.com/search?query=searchtext\n Got %s", url)); + } + sText = URLDecoder.decode(sText, StandardCharsets.UTF_8); + sText = sText.replaceAll("[^A-Za-z0-9_-]", "-"); + return sText; + } + m = isTags(); + if (m.matches()) { + var sText = m.group(1); + if (sText == null || sText.isBlank()){ + throw new MalformedURLException(String.format("Expected redgifs.com/gifs/searchtags\n Got %s", url)); + } + sText = URLDecoder.decode(sText, StandardCharsets.UTF_8); + var list = Arrays.asList(sText.split(",")); + if (list.size() > 1) { + logger.warn("Url with multiple tags found. \nThey will be sorted alphabetically for folder name."); + } + Collections.sort(list); + var gid = list.stream().reduce("", (acc, val) -> acc.concat("_" + val)); + gid = gid.replaceAll("[^A-Za-z0-9_-]", "-"); + return gid; } m = isSingleton(); if (m.matches()) { @@ -137,8 +172,10 @@ public class RedgifsRipper extends AbstractJSONRipper { } throw new MalformedURLException( "Expected redgifs.com format: " - + "redgifs.com/id or " - + "thumbs.redgifs.com/id.gif" + + "redgifs.com/watch/id or " + + "redgifs.com/users/id or " + + "redgifs.com/gifs/id or " + + "redgifs.com/search?query=text" + " Got: " + url); } @@ -160,10 +197,11 @@ public class RedgifsRipper extends AbstractJSONRipper { return null; } currentPage++; - if (isSearch().matches()) { - // TODO search - // TODO remove - throw new IOException("// TODO remove"); + if (isSearch().matches() || isTags().matches()) { + var json = Http.url(getSearchOrTagsURL()).header("Authorization", "Bearer " + authToken).getJSON(); + // Handle rare maxPages change during a rip + maxPages = json.getInt("pages"); + return json; } else if (isProfile().matches()) { var uri = new URIBuilder(String.format(USERS_SEARCH_ENDPOINT, getGID(url))); uri.addParameter("order", "new"); @@ -181,13 +219,12 @@ public class RedgifsRipper extends AbstractJSONRipper { @Override public List getURLsFromJSON(JSONObject json) { List result = new ArrayList<>(); - if (isProfile().matches() || isSearch().matches()) { - // TODO check json keys for search + if (isProfile().matches() || isSearch().matches() || isTags().matches()) { var gifs = json.getJSONArray("gifs"); for (var gif : gifs) { if (((JSONObject)gif).isNull("gallery")) { - var hdURL = ((JSONObject)gif).getJSONObject("urls").getString("hd"); - result.add(hdURL); + var hdURL = ((JSONObject)gif).getJSONObject("urls").getString("hd"); + result.add(hdURL); } else { var galleryID = ((JSONObject)gif).getString("gallery"); var gifID = ((JSONObject)gif).getString("id"); @@ -198,7 +235,7 @@ public class RedgifsRipper extends AbstractJSONRipper { var gif = json.getJSONObject("gif"); if (gif.isNull("gallery")) { String hdURL = gif.getJSONObject("urls").getString("hd"); - result.add(hdURL); + result.add(hdURL); } else { var galleryID = gif.getString("gallery"); var gifID = gif.getString("id"); @@ -285,4 +322,84 @@ public class RedgifsRipper extends AbstractJSONRipper { var token = json.getString("token"); authToken = token; } + + /** + * Map browser url query params to search or tags endpoint query params and return the complete url. + * + * Search text for search url comes from the query params, whereas search text for tags url comes from the path. + * + * Tab type for search url comes from the path whereas, tab type for tags url comes from query params. + * @return Search or tags endpoint url + */ + private URL getSearchOrTagsURL() throws IOException, URISyntaxException { + URIBuilder uri; + Map endpointQueryParams = new HashMap<>(); + var browserURLQueryParams = new URIBuilder(url.toString()).getQueryParams(); + for (var qp : browserURLQueryParams) { + var name = qp.getName(); + var value = qp.getValue(); + switch (name) { + case "query": + endpointQueryParams.put("query", URLDecoder.decode(value, StandardCharsets.UTF_8)); + break; + case "tab": + switch (value) { + case "gifs" -> endpointQueryParams.put("type", "g"); + case "images" -> endpointQueryParams.put("type", "i"); + default -> logger.warn(String.format("Unsupported tab for tags url %s", value)); + } + break; + case "verified": + if (value != null && value.equals("1")) { + if (isTags().matches()){ + endpointQueryParams.put("verified", "y"); + } else { + endpointQueryParams.put("verified", "yes"); + } + } + break; + case "order": + endpointQueryParams.put("order", value); + break; + case "viewMode": + break; + default: + logger.warn(String.format("Unexpected query param %s for search url. Skipping.", name)); + } + } + + // Build the search or tags url and add missing query params if any + if (isTags().matches()) { + var subpaths = url.getPath().split("/"); + if (subpaths.length != 0) { + endpointQueryParams.put("search_text", subpaths[subpaths.length-1]); + } else { + throw new IOException("Failed to get search tags for url"); + } + // Check if it is the main tags page with all gifs, images, creator etc + if (!endpointQueryParams.containsKey("type")) { + logger.warn("No tab selected, defaulting to gifs"); + endpointQueryParams.put("type", "g"); + } + uri = new URIBuilder(TAGS_ENDPOINT); + } else { + var tabType = "gifs"; + var subpaths = url.getPath().split("/"); + if (subpaths.length != 0) { + switch (subpaths[subpaths.length-1]) { + case "gifs" -> tabType = "gifs"; + case "images" -> tabType = "images"; + case "search" -> logger.warn("No tab selected, defaulting to gifs"); + default -> logger.warn(String.format("Unsupported search tab %s, defaulting to gifs", subpaths[subpaths.length-1])); + } + } + uri = new URIBuilder(String.format(SEARCH_ENDPOINT, tabType)); + } + + endpointQueryParams.put("page", Integer.toString(currentPage)); + endpointQueryParams.put("count", Integer.toString(count)); + endpointQueryParams.forEach((k, v) -> uri.addParameter(k, v)); + + return uri.build().toURL(); + } } From 8c455e7ec19945fd20c20bbb4477a8726b1a9a31 Mon Sep 17 00:00:00 2001 From: Tush-r Date: Thu, 29 Aug 2024 17:21:01 +0530 Subject: [PATCH 478/512] Refactor rgif ripper --- .../ripme/ripper/rippers/RedgifsRipper.java | 115 ++++++------------ 1 file changed, 37 insertions(+), 78 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java index 99a5c230..8dd10301 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java @@ -2,11 +2,7 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.utils.Http; -import org.json.JSONArray; import org.json.JSONObject; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import static com.rarchives.ripme.App.logger; @@ -40,19 +36,17 @@ public class RedgifsRipper extends AbstractJSONRipper { private static final String SEARCH_ENDPOINT = "https://api.redgifs.com/v2/search/%s"; private static final String TAGS_ENDPOINT = "https://api.redgifs.com/v2/gifs/search"; private static final String TEMPORARY_AUTH_ENDPOINT = "https://api.redgifs.com/v2/auth/temporary"; + private static final Pattern PROFILE_PATTERN = Pattern.compile("^https?://[wm.]*redgifs\\.com/users/([a-zA-Z0-9_.-]+).*$"); + private static final Pattern SEARCH_PATTERN = Pattern.compile("^https?:\\/\\/[wm.]*redgifs\\.com\\/search(?:\\/[a-zA-Z]+)?\\?.*?query=([a-zA-Z0-9-_+%]+).*$"); + private static final Pattern TAGS_PATTERN = Pattern.compile("^https?:\\/\\/[wm.]*redgifs\\.com\\/gifs\\/([a-zA-Z0-9_.,-]+).*$"); + private static final Pattern SINGLETON_PATTERN = Pattern.compile("^https?://[wm.]*redgifs\\.com/watch/([a-zA-Z0-9_-]+).*$"); + String username = ""; String authToken = ""; - // TODO remove - String cursor = ""; int count = 40; int currentPage = 1; int maxPages = 1; - // TODO remove with search - String searchText = ""; - int searchCount = 150; - int searchStart = 0; - public RedgifsRipper(URL url) throws IOException, URISyntaxException { super(new URI(url.toExternalForm().replace("thumbs.", "")).toURL()); } @@ -80,30 +74,26 @@ public class RedgifsRipper extends AbstractJSONRipper { } public Matcher isProfile() { - Pattern p = Pattern.compile("^https?://[wm.]*redgifs\\.com/users/([a-zA-Z0-9_.-]+).*$"); - return p.matcher(url.toExternalForm()); + return PROFILE_PATTERN.matcher(url.toExternalForm()); } public Matcher isSearch() { - Pattern p = Pattern.compile("^https?:\\/\\/[wm.]*redgifs\\.com\\/search(?:\\/[a-zA-Z]+)?\\?.*?query=([a-zA-Z0-9-_+%]+).*$"); - return p.matcher(url.toExternalForm()); + return SEARCH_PATTERN.matcher(url.toExternalForm()); } public Matcher isTags() { - Pattern p = Pattern.compile("^https?:\\/\\/[wm.]*redgifs\\.com\\/gifs\\/([a-zA-Z0-9_.,-]+).*$"); - return p.matcher(url.toExternalForm()); + return TAGS_PATTERN.matcher(url.toExternalForm()); } public Matcher isSingleton() { - Pattern p = Pattern.compile("^https?://[wm.]*redgifs\\.com/watch/([a-zA-Z0-9_-]+).*$"); - return p.matcher(url.toExternalForm()); + return SINGLETON_PATTERN.matcher(url.toExternalForm()); } @Override public JSONObject getFirstPage() throws IOException { try { - if (authToken == null || authToken.equals("")){ - fetchAuthToken(); + if (authToken == null || authToken.equals("")) { + authToken = fetchAuthToken(); } if (isSingleton().matches()) { @@ -143,7 +133,7 @@ public class RedgifsRipper extends AbstractJSONRipper { m = isSearch(); if (m.matches()) { var sText = m.group(1); - if (sText == null || sText.isBlank()){ + if (sText == null || sText.isBlank()) { throw new MalformedURLException(String.format("Expected redgifs.com/search?query=searchtext\n Got %s", url)); } sText = URLDecoder.decode(sText, StandardCharsets.UTF_8); @@ -153,7 +143,7 @@ public class RedgifsRipper extends AbstractJSONRipper { m = isTags(); if (m.matches()) { var sText = m.group(1); - if (sText == null || sText.isBlank()){ + if (sText == null || sText.isBlank()) { throw new MalformedURLException(String.format("Expected redgifs.com/gifs/searchtags\n Got %s", url)); } sText = URLDecoder.decode(sText, StandardCharsets.UTF_8); @@ -179,21 +169,9 @@ public class RedgifsRipper extends AbstractJSONRipper { + " Got: " + url); } - // TODO remove - private String stripHTMLTags(String t) { - t = t.replaceAll("\n" + - " \n" + - " ", ""); - t = t.replaceAll("\n" + - "", ""); - t = t.replaceAll("\n", ""); - t = t.replaceAll("=\"\"", ""); - return t; - } - @Override public JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException { - if (currentPage == maxPages || isSingleton().matches()){ + if (currentPage == maxPages || isSingleton().matches()) { return null; } currentPage++; @@ -228,7 +206,7 @@ public class RedgifsRipper extends AbstractJSONRipper { } else { var galleryID = ((JSONObject)gif).getString("gallery"); var gifID = ((JSONObject)gif).getString("id"); - result.addAll(getURLsForGallery(galleryID, gifID)); + result.addAll(getURLsForGallery(galleryID, gifID, authToken)); } } } else { @@ -239,7 +217,7 @@ public class RedgifsRipper extends AbstractJSONRipper { } else { var galleryID = gif.getString("gallery"); var gifID = gif.getString("id"); - result.addAll(getURLsForGallery(galleryID, gifID)); + result.addAll(getURLsForGallery(galleryID, gifID, authToken)); } } return result; @@ -252,7 +230,7 @@ public class RedgifsRipper extends AbstractJSONRipper { * @param gifID gif id with multiple images for logging * @return List */ - private List getURLsForGallery(String galleryID, String gifID) { + private static List getURLsForGallery(String galleryID, String gifID, String authToken) { List list = new ArrayList<>(); if (galleryID == null || galleryID.isBlank()) { return list; @@ -268,59 +246,40 @@ public class RedgifsRipper extends AbstractJSONRipper { } return list; } - - // TODO delete /** - * Helper method for retrieving URLs. - * @param doc Document of the URL page to look through - * @return List of URLs to download - */ - public List hasURLs(Document doc) { - List result = new ArrayList<>(); - JSONObject page = new JSONObject(stripHTMLTags(doc.html())); - JSONArray content = page.getJSONArray("gfycats"); - for (int i = 0; i < content.length(); i++) { - result.add(content.getJSONObject(i).getString("mp4Url")); - } - cursor = page.get("cursor").toString(); - return result; - } - - // TODO delete - /** - * Helper method for retrieving video URLs. - * @param url URL to gfycat page + * Static helper method for retrieving video URLs for usage in RipUtils. + * Most of the code is lifted from getFirstPage and getURLsFromJSON + * @param url URL to redgif page * @return URL to video * @throws IOException */ public static String getVideoURL(URL url) throws IOException, URISyntaxException { LOGGER.info("Retrieving " + url.toExternalForm()); - - //Sanitize the URL first - url = new URI(url.toExternalForm().replace("/gifs/detail", "")).toURL(); - - Document doc = Http.url(url).get(); - Elements videos = doc.select("script"); - for (Element el : videos) { - String json = el.html(); - if (json.startsWith("{")) { - JSONObject page = new JSONObject(json); - String mobileUrl = page.getJSONObject("video").getString("contentUrl"); - return mobileUrl.replace("-mobile", ""); - } + var m = SINGLETON_PATTERN.matcher(url.toExternalForm()); + if (!m.matches()){ + throw new IOException(String.format("Cannot fetch redgif url %s", url.toExternalForm())); } - throw new IOException(); + var authToken = fetchAuthToken(); + var gid = m.group(1).split("-")[0]; + var gifDetailsURL = String.format(GIFS_DETAIL_ENDPOINT, gid); + var json = Http.url(gifDetailsURL).header("Authorization", "Bearer " + authToken).getJSON(); + var gif = json.getJSONObject("gif"); + if (!gif.isNull("gallery")){ + // TODO check how to handle a image gallery + throw new IOException(String.format("Multiple images found for url %s", url)); + } + return gif.getJSONObject("urls").getString("hd"); } - + /** * Fetch a temorary auth token for the rip * @throws IOException */ - private void fetchAuthToken() throws IOException{ + private static String fetchAuthToken() throws IOException{ var json = Http.url(TEMPORARY_AUTH_ENDPOINT).getJSON(); var token = json.getString("token"); - authToken = token; + return token; } /** @@ -351,7 +310,7 @@ public class RedgifsRipper extends AbstractJSONRipper { break; case "verified": if (value != null && value.equals("1")) { - if (isTags().matches()){ + if (isTags().matches()) { endpointQueryParams.put("verified", "y"); } else { endpointQueryParams.put("verified", "yes"); From a7cb78eceb16e910dbbedf66b4e2cdf52445d581 Mon Sep 17 00:00:00 2001 From: Tush-r Date: Thu, 29 Aug 2024 17:21:01 +0530 Subject: [PATCH 479/512] Fix logger for rgif ripper --- .../ripme/ripper/rippers/RedgifsRipper.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java index 8dd10301..a193d2e6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java @@ -4,8 +4,6 @@ import com.rarchives.ripme.utils.Http; import org.json.JSONObject; -import static com.rarchives.ripme.App.logger; - import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; @@ -149,7 +147,7 @@ public class RedgifsRipper extends AbstractJSONRipper { sText = URLDecoder.decode(sText, StandardCharsets.UTF_8); var list = Arrays.asList(sText.split(",")); if (list.size() > 1) { - logger.warn("Url with multiple tags found. \nThey will be sorted alphabetically for folder name."); + LOGGER.warn("Url with multiple tags found. \nThey will be sorted alphabetically for folder name."); } Collections.sort(list); var gid = list.stream().reduce("", (acc, val) -> acc.concat("_" + val)); @@ -242,7 +240,7 @@ public class RedgifsRipper extends AbstractJSONRipper { list.add(hdURL); } } catch (IOException e) { - logger.error(String.format("Error fetching gallery %s for gif %s", galleryID, gifID), e); + LOGGER.error(String.format("Error fetching gallery %s for gif %s", galleryID, gifID), e); } return list; } @@ -305,7 +303,7 @@ public class RedgifsRipper extends AbstractJSONRipper { switch (value) { case "gifs" -> endpointQueryParams.put("type", "g"); case "images" -> endpointQueryParams.put("type", "i"); - default -> logger.warn(String.format("Unsupported tab for tags url %s", value)); + default -> LOGGER.warn(String.format("Unsupported tab for tags url %s", value)); } break; case "verified": @@ -323,7 +321,7 @@ public class RedgifsRipper extends AbstractJSONRipper { case "viewMode": break; default: - logger.warn(String.format("Unexpected query param %s for search url. Skipping.", name)); + LOGGER.warn(String.format("Unexpected query param %s for search url. Skipping.", name)); } } @@ -337,7 +335,7 @@ public class RedgifsRipper extends AbstractJSONRipper { } // Check if it is the main tags page with all gifs, images, creator etc if (!endpointQueryParams.containsKey("type")) { - logger.warn("No tab selected, defaulting to gifs"); + LOGGER.warn("No tab selected, defaulting to gifs"); endpointQueryParams.put("type", "g"); } uri = new URIBuilder(TAGS_ENDPOINT); @@ -348,8 +346,8 @@ public class RedgifsRipper extends AbstractJSONRipper { switch (subpaths[subpaths.length-1]) { case "gifs" -> tabType = "gifs"; case "images" -> tabType = "images"; - case "search" -> logger.warn("No tab selected, defaulting to gifs"); - default -> logger.warn(String.format("Unsupported search tab %s, defaulting to gifs", subpaths[subpaths.length-1])); + case "search" -> LOGGER.warn("No tab selected, defaulting to gifs"); + default -> LOGGER.warn(String.format("Unsupported search tab %s, defaulting to gifs", subpaths[subpaths.length-1])); } } uri = new URIBuilder(String.format(SEARCH_ENDPOINT, tabType)); From 99a5d88854b844c576d2c8196968cc07fa663ecb Mon Sep 17 00:00:00 2001 From: Tush-r Date: Thu, 29 Aug 2024 17:21:01 +0530 Subject: [PATCH 480/512] Fix rgif ripper tests --- .../tst/ripper/rippers/RedgifsRipperTest.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java index f89f9c40..3ef0759c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java @@ -2,7 +2,6 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.RedditRipper; import com.rarchives.ripme.ripper.rippers.RedgifsRipper; -import org.jsoup.nodes.Document; import org.junit.jupiter.api.*; import java.io.IOException; @@ -15,9 +14,8 @@ public class RedgifsRipperTest extends RippersTest { * Rips correctly formatted URL directly from Redgifs */ @Test - @Disabled("test or ripper broken") public void testRedgifsGoodURL() throws IOException, URISyntaxException { - RedgifsRipper ripper = new RedgifsRipper(new URI("https://www.redgifs.com/watch/talkativewarpeddragon-petite").toURL()); + RedgifsRipper ripper = new RedgifsRipper(new URI("https://www.redgifs.com/watch/ashamedselfishcoypu").toURL()); testRipper(ripper); } @@ -25,36 +23,38 @@ public class RedgifsRipperTest extends RippersTest { * Rips gifdeliverynetwork URL's by redirecting them to proper redgifs url */ @Test - @Tag("flaky") public void testRedgifsBadRL() throws IOException, URISyntaxException { - RedgifsRipper ripper = new RedgifsRipper(new URI("https://www.gifdeliverynetwork.com/foolishelasticchimpanzee").toURL()); + RedgifsRipper ripper = new RedgifsRipper(new URI("https://www.gifdeliverynetwork.com/consideratetrustworthypigeon").toURL()); testRipper(ripper); } /** - * Rips a Redifs profile + * Rips a Redgifs profile */ @Test - @Tag("flaky") public void testRedgifsProfile() throws IOException, URISyntaxException { - RedgifsRipper ripper = new RedgifsRipper(new URI("https://redgifs.com/users/margo_monty").toURL()); + RedgifsRipper ripper = new RedgifsRipper(new URI("https://www.redgifs.com/users/ra-kunv2").toURL()); testRipper(ripper); } /** - * Rips a Redifs category/search + * Rips a Redgif search * @throws IOException */ @Test - @Disabled("test or ripper broken") public void testRedgifsSearch() throws IOException, URISyntaxException { - RedgifsRipper ripper = new RedgifsRipper(new URI("https://redgifs.com/gifs/browse/little-caprice").toURL()); - Document doc = ripper.getFirstPage(); + RedgifsRipper ripper = new RedgifsRipper(new URI("https://www.redgifs.com/search?query=take+a+shot+every+time").toURL()); + testRipper(ripper); + } - doc = ripper.getNextPage(doc); - Assertions.assertTrue("https://api.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=150".equalsIgnoreCase(doc.location())); - doc = ripper.getNextPage(doc); - Assertions.assertTrue("https://api.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=300".equalsIgnoreCase(doc.location())); + /** + * Rips Redgif tags + * @throws IOException + */ + @Test + public void testRedgifsTags() throws IOException, URISyntaxException { + RedgifsRipper ripper = new RedgifsRipper(new URI("https://www.redgifs.com/gifs/animation,sfw,funny?order=best&tab=gifs").toURL()); + testRipper(ripper); } @Test From 8d27571e230f317b0a1645b321f34678228f121c Mon Sep 17 00:00:00 2001 From: Tush-r Date: Thu, 29 Aug 2024 17:21:01 +0530 Subject: [PATCH 481/512] Add support for rgif v3 urls --- .../com/rarchives/ripme/ripper/rippers/RedgifsRipper.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java index a193d2e6..b54bcba8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java @@ -34,10 +34,10 @@ public class RedgifsRipper extends AbstractJSONRipper { private static final String SEARCH_ENDPOINT = "https://api.redgifs.com/v2/search/%s"; private static final String TAGS_ENDPOINT = "https://api.redgifs.com/v2/gifs/search"; private static final String TEMPORARY_AUTH_ENDPOINT = "https://api.redgifs.com/v2/auth/temporary"; - private static final Pattern PROFILE_PATTERN = Pattern.compile("^https?://[wm.]*redgifs\\.com/users/([a-zA-Z0-9_.-]+).*$"); - private static final Pattern SEARCH_PATTERN = Pattern.compile("^https?:\\/\\/[wm.]*redgifs\\.com\\/search(?:\\/[a-zA-Z]+)?\\?.*?query=([a-zA-Z0-9-_+%]+).*$"); - private static final Pattern TAGS_PATTERN = Pattern.compile("^https?:\\/\\/[wm.]*redgifs\\.com\\/gifs\\/([a-zA-Z0-9_.,-]+).*$"); - private static final Pattern SINGLETON_PATTERN = Pattern.compile("^https?://[wm.]*redgifs\\.com/watch/([a-zA-Z0-9_-]+).*$"); + private static final Pattern PROFILE_PATTERN = Pattern.compile("^https?://[a-zA-Z0-9.]*redgifs\\.com/users/([a-zA-Z0-9_.-]+).*$"); + private static final Pattern SEARCH_PATTERN = Pattern.compile("^https?:\\/\\/[a-zA-Z0-9.]*redgifs\\.com\\/search(?:\\/[a-zA-Z]+)?\\?.*?query=([a-zA-Z0-9-_+%]+).*$"); + private static final Pattern TAGS_PATTERN = Pattern.compile("^https?:\\/\\/[a-zA-Z0-9.]*redgifs\\.com\\/gifs\\/([a-zA-Z0-9_.,-]+).*$"); + private static final Pattern SINGLETON_PATTERN = Pattern.compile("^https?://[a-zA-Z0-9.]*redgifs\\.com/watch/([a-zA-Z0-9_-]+).*$"); String username = ""; String authToken = ""; From f39c8b9395a10ab85be7e77c3174d528dac92152 Mon Sep 17 00:00:00 2001 From: Tush-r Date: Thu, 29 Aug 2024 17:45:48 +0530 Subject: [PATCH 482/512] Set static auth token for redgif --- .../ripme/ripper/rippers/RedgifsRipper.java | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java index b54bcba8..e82db4b2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java @@ -39,8 +39,13 @@ public class RedgifsRipper extends AbstractJSONRipper { private static final Pattern TAGS_PATTERN = Pattern.compile("^https?:\\/\\/[a-zA-Z0-9.]*redgifs\\.com\\/gifs\\/([a-zA-Z0-9_.,-]+).*$"); private static final Pattern SINGLETON_PATTERN = Pattern.compile("^https?://[a-zA-Z0-9.]*redgifs\\.com/watch/([a-zA-Z0-9_-]+).*$"); + /** + * Keep a single auth token for the complete lifecycle of the app. + * This should prevent fetching of multiple tokens. + */ + private static String authToken = ""; + String username = ""; - String authToken = ""; int count = 40; int currentPage = 1; int maxPages = 1; @@ -90,8 +95,8 @@ public class RedgifsRipper extends AbstractJSONRipper { @Override public JSONObject getFirstPage() throws IOException { try { - if (authToken == null || authToken.equals("")) { - authToken = fetchAuthToken(); + if (authToken == null || authToken.isBlank()) { + fetchAuthToken(); } if (isSingleton().matches()) { @@ -204,7 +209,7 @@ public class RedgifsRipper extends AbstractJSONRipper { } else { var galleryID = ((JSONObject)gif).getString("gallery"); var gifID = ((JSONObject)gif).getString("id"); - result.addAll(getURLsForGallery(galleryID, gifID, authToken)); + result.addAll(getURLsForGallery(galleryID, gifID)); } } } else { @@ -215,7 +220,7 @@ public class RedgifsRipper extends AbstractJSONRipper { } else { var galleryID = gif.getString("gallery"); var gifID = gif.getString("id"); - result.addAll(getURLsForGallery(galleryID, gifID, authToken)); + result.addAll(getURLsForGallery(galleryID, gifID)); } } return result; @@ -228,7 +233,7 @@ public class RedgifsRipper extends AbstractJSONRipper { * @param gifID gif id with multiple images for logging * @return List */ - private static List getURLsForGallery(String galleryID, String gifID, String authToken) { + private static List getURLsForGallery(String galleryID, String gifID) { List list = new ArrayList<>(); if (galleryID == null || galleryID.isBlank()) { return list; @@ -257,7 +262,9 @@ public class RedgifsRipper extends AbstractJSONRipper { if (!m.matches()){ throw new IOException(String.format("Cannot fetch redgif url %s", url.toExternalForm())); } - var authToken = fetchAuthToken(); + if (authToken == null || authToken.isBlank()){ + fetchAuthToken(); + } var gid = m.group(1).split("-")[0]; var gifDetailsURL = String.format(GIFS_DETAIL_ENDPOINT, gid); var json = Http.url(gifDetailsURL).header("Authorization", "Bearer " + authToken).getJSON(); @@ -274,10 +281,11 @@ public class RedgifsRipper extends AbstractJSONRipper { * Fetch a temorary auth token for the rip * @throws IOException */ - private static String fetchAuthToken() throws IOException{ + private static void fetchAuthToken() throws IOException{ var json = Http.url(TEMPORARY_AUTH_ENDPOINT).getJSON(); var token = json.getString("token"); - return token; + authToken = token; + LOGGER.info("Incase of redgif 401 errors, please restart the app to refresh the auth token"); } /** From c94a9543b4e57b54d89003430865e291c7442ca3 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 8 Sep 2024 16:01:27 +0200 Subject: [PATCH 483/512] Furufanity test flaky --- .../ripme/tst/ripper/rippers/FuraffinityRipperTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java index 87946d1d..fd21aff8 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FuraffinityRipperTest.java @@ -31,6 +31,7 @@ public class FuraffinityRipperTest extends RippersTest { Assertions.assertEquals("mustardgas", ripper.getGID(url)); } @Test + @Tag("flaky") public void testLogin() throws IOException, URISyntaxException { URL url = new URI("https://www.furaffinity.net/gallery/mustardgas/").toURL(); FuraffinityRipper ripper = new FuraffinityRipper(url); From 7939bf9ec159afb26867b9399eda5735c94ae84d Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 8 Sep 2024 17:03:33 +0200 Subject: [PATCH 484/512] release 2.1.10 --- ripme.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ripme.json b/ripme.json index c2f54680..c3a90ca0 100644 --- a/ripme.json +++ b/ripme.json @@ -1,7 +1,8 @@ { - "latestVersion": "2.1.9-7-22e915df", + "latestVersion": "2.1.10-21-c94a9543", "currentHash": "782ffec29bd14cfde6d714fa6f76980b3fd7cf96723b1121976134a6a5057e68", "changeList": [ + "2.1.10-21-c94a9543, Imagebam, Unify colons in UI, Motherless, right click menu, rgif fixed", "2.1.9-7-22e915df, HistoryMenuMouseListener right click menu, Imagefap retry logic for getFullSizedImage(), EightmusesRipper fixed", "2.1.8-1-f5153de8: jpg3 add, java-21 adjustments.", "2.1.7-29-b080faae: luciousripper fix, java-21 adjustments.", From 31c71f4484aab2a077a92c274e3938b204c5012b Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 8 Sep 2024 17:09:55 +0200 Subject: [PATCH 485/512] upgrade to gradle-8.10 --- gradle/wrapper/gradle-wrapper.jar | Bin 63721 -> 43583 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 7 +++++-- gradlew.bat | 22 ++++++++++++---------- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 7f93135c49b765f8051ef9d0a6055ff8e46073d8..a4b76b9530d66f5e68d973ea569d8e19de379189 100644 GIT binary patch literal 43583 zcma&N1CXTcmMvW9vTb(Rwr$&4wr$(C?dmSu>@vG-+vuvg^_??!{yS%8zW-#zn-LkA z5&1^$^{lnmUON?}LBF8_K|(?T0Ra(xUH{($5eN!MR#ZihR#HxkUPe+_R8Cn`RRs(P z_^*#_XlXmGv7!4;*Y%p4nw?{bNp@UZHv1?Um8r6)Fei3p@ClJn0ECfg1hkeuUU@Or zDaPa;U3fE=3L}DooL;8f;P0ipPt0Z~9P0)lbStMS)ag54=uL9ia-Lm3nh|@(Y?B`; zx_#arJIpXH!U{fbCbI^17}6Ri*H<>OLR%c|^mh8+)*h~K8Z!9)DPf zR2h?lbDZQ`p9P;&DQ4F0sur@TMa!Y}S8irn(%d-gi0*WxxCSk*A?3lGh=gcYN?FGl z7D=Js!i~0=u3rox^eO3i@$0=n{K1lPNU zwmfjRVmLOCRfe=seV&P*1Iq=^i`502keY8Uy-WNPwVNNtJFx?IwAyRPZo2Wo1+S(xF37LJZ~%i)kpFQ3Fw=mXfd@>%+)RpYQLnr}B~~zoof(JVm^^&f zxKV^+3D3$A1G;qh4gPVjhrC8e(VYUHv#dy^)(RoUFM?o%W-EHxufuWf(l*@-l+7vt z=l`qmR56K~F|v<^Pd*p~1_y^P0P^aPC##d8+HqX4IR1gu+7w#~TBFphJxF)T$2WEa zxa?H&6=Qe7d(#tha?_1uQys2KtHQ{)Qco)qwGjrdNL7thd^G5i8Os)CHqc>iOidS} z%nFEDdm=GXBw=yXe1W-ShHHFb?Cc70+$W~z_+}nAoHFYI1MV1wZegw*0y^tC*s%3h zhD3tN8b=Gv&rj}!SUM6|ajSPp*58KR7MPpI{oAJCtY~JECm)*m_x>AZEu>DFgUcby z1Qaw8lU4jZpQ_$;*7RME+gq1KySGG#Wql>aL~k9tLrSO()LWn*q&YxHEuzmwd1?aAtI zBJ>P=&$=l1efe1CDU;`Fd+_;&wI07?V0aAIgc(!{a z0Jg6Y=inXc3^n!U0Atk`iCFIQooHqcWhO(qrieUOW8X(x?(RD}iYDLMjSwffH2~tB z)oDgNBLB^AJBM1M^c5HdRx6fBfka`(LD-qrlh5jqH~);#nw|iyp)()xVYak3;Ybik z0j`(+69aK*B>)e_p%=wu8XC&9e{AO4c~O1U`5X9}?0mrd*m$_EUek{R?DNSh(=br# z#Q61gBzEpmy`$pA*6!87 zSDD+=@fTY7<4A?GLqpA?Pb2z$pbCc4B4zL{BeZ?F-8`s$?>*lXXtn*NC61>|*w7J* z$?!iB{6R-0=KFmyp1nnEmLsA-H0a6l+1uaH^g%c(p{iT&YFrbQ$&PRb8Up#X3@Zsk zD^^&LK~111%cqlP%!_gFNa^dTYT?rhkGl}5=fL{a`UViaXWI$k-UcHJwmaH1s=S$4 z%4)PdWJX;hh5UoK?6aWoyLxX&NhNRqKam7tcOkLh{%j3K^4Mgx1@i|Pi&}<^5>hs5 zm8?uOS>%)NzT(%PjVPGa?X%`N2TQCKbeH2l;cTnHiHppPSJ<7y-yEIiC!P*ikl&!B z%+?>VttCOQM@ShFguHVjxX^?mHX^hSaO_;pnyh^v9EumqSZTi+#f&_Vaija0Q-e*| z7ulQj6Fs*bbmsWp{`auM04gGwsYYdNNZcg|ph0OgD>7O}Asn7^Z=eI>`$2*v78;sj-}oMoEj&@)9+ycEOo92xSyY344^ z11Hb8^kdOvbf^GNAK++bYioknrpdN>+u8R?JxG=!2Kd9r=YWCOJYXYuM0cOq^FhEd zBg2puKy__7VT3-r*dG4c62Wgxi52EMCQ`bKgf*#*ou(D4-ZN$+mg&7$u!! z-^+Z%;-3IDwqZ|K=ah85OLwkO zKxNBh+4QHh)u9D?MFtpbl)us}9+V!D%w9jfAMYEb>%$A;u)rrI zuBudh;5PN}_6J_}l55P3l_)&RMlH{m!)ai-i$g)&*M`eN$XQMw{v^r@-125^RRCF0 z^2>|DxhQw(mtNEI2Kj(;KblC7x=JlK$@78`O~>V!`|1Lm-^JR$-5pUANAnb(5}B}JGjBsliK4& zk6y(;$e&h)lh2)L=bvZKbvh@>vLlreBdH8No2>$#%_Wp1U0N7Ank!6$dFSi#xzh|( zRi{Uw%-4W!{IXZ)fWx@XX6;&(m_F%c6~X8hx=BN1&q}*( zoaNjWabE{oUPb!Bt$eyd#$5j9rItB-h*5JiNi(v^e|XKAj*8(k<5-2$&ZBR5fF|JA z9&m4fbzNQnAU}r8ab>fFV%J0z5awe#UZ|bz?Ur)U9bCIKWEzi2%A+5CLqh?}K4JHi z4vtM;+uPsVz{Lfr;78W78gC;z*yTch~4YkLr&m-7%-xc ztw6Mh2d>_iO*$Rd8(-Cr1_V8EO1f*^@wRoSozS) zy1UoC@pruAaC8Z_7~_w4Q6n*&B0AjOmMWa;sIav&gu z|J5&|{=a@vR!~k-OjKEgPFCzcJ>#A1uL&7xTDn;{XBdeM}V=l3B8fE1--DHjSaxoSjNKEM9|U9#m2<3>n{Iuo`r3UZp;>GkT2YBNAh|b z^jTq-hJp(ebZh#Lk8hVBP%qXwv-@vbvoREX$TqRGTgEi$%_F9tZES@z8Bx}$#5eeG zk^UsLBH{bc2VBW)*EdS({yw=?qmevwi?BL6*=12k9zM5gJv1>y#ML4!)iiPzVaH9% zgSImetD@dam~e>{LvVh!phhzpW+iFvWpGT#CVE5TQ40n%F|p(sP5mXxna+Ev7PDwA zamaV4m*^~*xV+&p;W749xhb_X=$|LD;FHuB&JL5?*Y2-oIT(wYY2;73<^#46S~Gx| z^cez%V7x$81}UWqS13Gz80379Rj;6~WdiXWOSsdmzY39L;Hg3MH43o*y8ibNBBH`(av4|u;YPq%{R;IuYow<+GEsf@R?=@tT@!}?#>zIIn0CoyV!hq3mw zHj>OOjfJM3F{RG#6ujzo?y32m^tgSXf@v=J$ELdJ+=5j|=F-~hP$G&}tDZsZE?5rX ztGj`!S>)CFmdkccxM9eGIcGnS2AfK#gXwj%esuIBNJQP1WV~b~+D7PJTmWGTSDrR` zEAu4B8l>NPuhsk5a`rReSya2nfV1EK01+G!x8aBdTs3Io$u5!6n6KX%uv@DxAp3F@{4UYg4SWJtQ-W~0MDb|j-$lwVn znAm*Pl!?Ps&3wO=R115RWKb*JKoexo*)uhhHBncEDMSVa_PyA>k{Zm2(wMQ(5NM3# z)jkza|GoWEQo4^s*wE(gHz?Xsg4`}HUAcs42cM1-qq_=+=!Gk^y710j=66(cSWqUe zklbm8+zB_syQv5A2rj!Vbw8;|$@C!vfNmNV!yJIWDQ>{+2x zKjuFX`~~HKG~^6h5FntRpnnHt=D&rq0>IJ9#F0eM)Y-)GpRjiN7gkA8wvnG#K=q{q z9dBn8_~wm4J<3J_vl|9H{7q6u2A!cW{bp#r*-f{gOV^e=8S{nc1DxMHFwuM$;aVI^ zz6A*}m8N-&x8;aunp1w7_vtB*pa+OYBw=TMc6QK=mbA-|Cf* zvyh8D4LRJImooUaSb7t*fVfih<97Gf@VE0|z>NcBwBQze);Rh!k3K_sfunToZY;f2 z^HmC4KjHRVg+eKYj;PRN^|E0>Gj_zagfRbrki68I^#~6-HaHg3BUW%+clM1xQEdPYt_g<2K+z!$>*$9nQ>; zf9Bei{?zY^-e{q_*|W#2rJG`2fy@{%6u0i_VEWTq$*(ZN37|8lFFFt)nCG({r!q#9 z5VK_kkSJ3?zOH)OezMT{!YkCuSSn!K#-Rhl$uUM(bq*jY? zi1xbMVthJ`E>d>(f3)~fozjg^@eheMF6<)I`oeJYx4*+M&%c9VArn(OM-wp%M<-`x z7sLP1&3^%Nld9Dhm@$3f2}87!quhI@nwd@3~fZl_3LYW-B?Ia>ui`ELg z&Qfe!7m6ze=mZ`Ia9$z|ARSw|IdMpooY4YiPN8K z4B(ts3p%2i(Td=tgEHX z0UQ_>URBtG+-?0E;E7Ld^dyZ;jjw0}XZ(}-QzC6+NN=40oDb2^v!L1g9xRvE#@IBR zO!b-2N7wVfLV;mhEaXQ9XAU+>=XVA6f&T4Z-@AX!leJ8obP^P^wP0aICND?~w&NykJ#54x3_@r7IDMdRNy4Hh;h*!u(Ol(#0bJdwEo$5437-UBjQ+j=Ic>Q2z` zJNDf0yO6@mr6y1#n3)s(W|$iE_i8r@Gd@!DWDqZ7J&~gAm1#~maIGJ1sls^gxL9LLG_NhU!pTGty!TbhzQnu)I*S^54U6Yu%ZeCg`R>Q zhBv$n5j0v%O_j{QYWG!R9W?5_b&67KB$t}&e2LdMvd(PxN6Ir!H4>PNlerpBL>Zvyy!yw z-SOo8caEpDt(}|gKPBd$qND5#a5nju^O>V&;f890?yEOfkSG^HQVmEbM3Ugzu+UtH zC(INPDdraBN?P%kE;*Ae%Wto&sgw(crfZ#Qy(<4nk;S|hD3j{IQRI6Yq|f^basLY; z-HB&Je%Gg}Jt@={_C{L$!RM;$$|iD6vu#3w?v?*;&()uB|I-XqEKqZPS!reW9JkLewLb!70T7n`i!gNtb1%vN- zySZj{8-1>6E%H&=V}LM#xmt`J3XQoaD|@XygXjdZ1+P77-=;=eYpoEQ01B@L*a(uW zrZeZz?HJsw_4g0vhUgkg@VF8<-X$B8pOqCuWAl28uB|@r`19DTUQQsb^pfqB6QtiT z*`_UZ`fT}vtUY#%sq2{rchyfu*pCg;uec2$-$N_xgjZcoumE5vSI{+s@iLWoz^Mf; zuI8kDP{!XY6OP~q5}%1&L}CtfH^N<3o4L@J@zg1-mt{9L`s^z$Vgb|mr{@WiwAqKg zp#t-lhrU>F8o0s1q_9y`gQNf~Vb!F%70f}$>i7o4ho$`uciNf=xgJ>&!gSt0g;M>*x4-`U)ysFW&Vs^Vk6m%?iuWU+o&m(2Jm26Y(3%TL; zA7T)BP{WS!&xmxNw%J=$MPfn(9*^*TV;$JwRy8Zl*yUZi8jWYF>==j~&S|Xinsb%c z2?B+kpet*muEW7@AzjBA^wAJBY8i|#C{WtO_or&Nj2{=6JTTX05}|H>N2B|Wf!*3_ z7hW*j6p3TvpghEc6-wufFiY!%-GvOx*bZrhZu+7?iSrZL5q9}igiF^*R3%DE4aCHZ zqu>xS8LkW+Auv%z-<1Xs92u23R$nk@Pk}MU5!gT|c7vGlEA%G^2th&Q*zfg%-D^=f z&J_}jskj|Q;73NP4<4k*Y%pXPU2Thoqr+5uH1yEYM|VtBPW6lXaetokD0u z9qVek6Q&wk)tFbQ8(^HGf3Wp16gKmr>G;#G(HRBx?F`9AIRboK+;OfHaLJ(P>IP0w zyTbTkx_THEOs%Q&aPrxbZrJlio+hCC_HK<4%f3ZoSAyG7Dn`=X=&h@m*|UYO-4Hq0 z-Bq&+Ie!S##4A6OGoC~>ZW`Y5J)*ouaFl_e9GA*VSL!O_@xGiBw!AF}1{tB)z(w%c zS1Hmrb9OC8>0a_$BzeiN?rkPLc9%&;1CZW*4}CDDNr2gcl_3z+WC15&H1Zc2{o~i) z)LLW=WQ{?ricmC`G1GfJ0Yp4Dy~Ba;j6ZV4r{8xRs`13{dD!xXmr^Aga|C=iSmor% z8hi|pTXH)5Yf&v~exp3o+sY4B^^b*eYkkCYl*T{*=-0HniSA_1F53eCb{x~1k3*`W zr~};p1A`k{1DV9=UPnLDgz{aJH=-LQo<5%+Em!DNN252xwIf*wF_zS^!(XSm(9eoj z=*dXG&n0>)_)N5oc6v!>-bd(2ragD8O=M|wGW z!xJQS<)u70m&6OmrF0WSsr@I%T*c#Qo#Ha4d3COcX+9}hM5!7JIGF>7<~C(Ear^Sn zm^ZFkV6~Ula6+8S?oOROOA6$C&q&dp`>oR-2Ym3(HT@O7Sd5c~+kjrmM)YmgPH*tL zX+znN>`tv;5eOfX?h{AuX^LK~V#gPCu=)Tigtq9&?7Xh$qN|%A$?V*v=&-2F$zTUv z`C#WyIrChS5|Kgm_GeudCFf;)!WH7FI60j^0o#65o6`w*S7R@)88n$1nrgU(oU0M9 zx+EuMkC>(4j1;m6NoGqEkpJYJ?vc|B zOlwT3t&UgL!pX_P*6g36`ZXQ; z9~Cv}ANFnJGp(;ZhS(@FT;3e)0)Kp;h^x;$*xZn*k0U6-&FwI=uOGaODdrsp-!K$Ac32^c{+FhI-HkYd5v=`PGsg%6I`4d9Jy)uW0y%) zm&j^9WBAp*P8#kGJUhB!L?a%h$hJgQrx!6KCB_TRo%9{t0J7KW8!o1B!NC)VGLM5! zpZy5Jc{`r{1e(jd%jsG7k%I+m#CGS*BPA65ZVW~fLYw0dA-H_}O zrkGFL&P1PG9p2(%QiEWm6x;U-U&I#;Em$nx-_I^wtgw3xUPVVu zqSuKnx&dIT-XT+T10p;yjo1Y)z(x1fb8Dzfn8e yu?e%!_ptzGB|8GrCfu%p?(_ zQccdaaVK$5bz;*rnyK{_SQYM>;aES6Qs^lj9lEs6_J+%nIiuQC*fN;z8md>r_~Mfl zU%p5Dt_YT>gQqfr@`cR!$NWr~+`CZb%dn;WtzrAOI>P_JtsB76PYe*<%H(y>qx-`Kq!X_; z<{RpAqYhE=L1r*M)gNF3B8r(<%8mo*SR2hu zccLRZwGARt)Hlo1euqTyM>^!HK*!Q2P;4UYrysje@;(<|$&%vQekbn|0Ruu_Io(w4#%p6ld2Yp7tlA`Y$cciThP zKzNGIMPXX%&Ud0uQh!uQZz|FB`4KGD?3!ND?wQt6!n*f4EmCoJUh&b?;B{|lxs#F- z31~HQ`SF4x$&v00@(P+j1pAaj5!s`)b2RDBp*PB=2IB>oBF!*6vwr7Dp%zpAx*dPr zb@Zjq^XjN?O4QcZ*O+8>)|HlrR>oD*?WQl5ri3R#2?*W6iJ>>kH%KnnME&TT@ZzrHS$Q%LC?n|e>V+D+8D zYc4)QddFz7I8#}y#Wj6>4P%34dZH~OUDb?uP%-E zwjXM(?Sg~1!|wI(RVuxbu)-rH+O=igSho_pDCw(c6b=P zKk4ATlB?bj9+HHlh<_!&z0rx13K3ZrAR8W)!@Y}o`?a*JJsD+twZIv`W)@Y?Amu_u zz``@-e2X}27$i(2=9rvIu5uTUOVhzwu%mNazS|lZb&PT;XE2|B&W1>=B58#*!~D&) zfVmJGg8UdP*fx(>Cj^?yS^zH#o-$Q-*$SnK(ZVFkw+er=>N^7!)FtP3y~Xxnu^nzY zikgB>Nj0%;WOltWIob|}%lo?_C7<``a5hEkx&1ku$|)i>Rh6@3h*`slY=9U}(Ql_< zaNG*J8vb&@zpdhAvv`?{=zDedJ23TD&Zg__snRAH4eh~^oawdYi6A3w8<Ozh@Kw)#bdktM^GVb zrG08?0bG?|NG+w^&JvD*7LAbjED{_Zkc`3H!My>0u5Q}m!+6VokMLXxl`Mkd=g&Xx z-a>m*#G3SLlhbKB!)tnzfWOBV;u;ftU}S!NdD5+YtOjLg?X}dl>7m^gOpihrf1;PY zvll&>dIuUGs{Qnd- zwIR3oIrct8Va^Tm0t#(bJD7c$Z7DO9*7NnRZorrSm`b`cxz>OIC;jSE3DO8`hX955ui`s%||YQtt2 z5DNA&pG-V+4oI2s*x^>-$6J?p=I>C|9wZF8z;VjR??Icg?1w2v5Me+FgAeGGa8(3S z4vg*$>zC-WIVZtJ7}o9{D-7d>zCe|z#<9>CFve-OPAYsneTb^JH!Enaza#j}^mXy1 z+ULn^10+rWLF6j2>Ya@@Kq?26>AqK{A_| zQKb*~F1>sE*=d?A?W7N2j?L09_7n+HGi{VY;MoTGr_)G9)ot$p!-UY5zZ2Xtbm=t z@dpPSGwgH=QtIcEulQNI>S-#ifbnO5EWkI;$A|pxJd885oM+ zGZ0_0gDvG8q2xebj+fbCHYfAXuZStH2j~|d^sBAzo46(K8n59+T6rzBwK)^rfPT+B zyIFw)9YC-V^rhtK`!3jrhmW-sTmM+tPH+;nwjL#-SjQPUZ53L@A>y*rt(#M(qsiB2 zx6B)dI}6Wlsw%bJ8h|(lhkJVogQZA&n{?Vgs6gNSXzuZpEyu*xySy8ro07QZ7Vk1!3tJphN_5V7qOiyK8p z#@jcDD8nmtYi1^l8ml;AF<#IPK?!pqf9D4moYk>d99Im}Jtwj6c#+A;f)CQ*f-hZ< z=p_T86jog%!p)D&5g9taSwYi&eP z#JuEK%+NULWus;0w32-SYFku#i}d~+{Pkho&^{;RxzP&0!RCm3-9K6`>KZpnzS6?L z^H^V*s!8<>x8bomvD%rh>Zp3>Db%kyin;qtl+jAv8Oo~1g~mqGAC&Qi_wy|xEt2iz zWAJEfTV%cl2Cs<1L&DLRVVH05EDq`pH7Oh7sR`NNkL%wi}8n>IXcO40hp+J+sC!W?!krJf!GJNE8uj zg-y~Ns-<~D?yqbzVRB}G>0A^f0!^N7l=$m0OdZuqAOQqLc zX?AEGr1Ht+inZ-Qiwnl@Z0qukd__a!C*CKuGdy5#nD7VUBM^6OCpxCa2A(X;e0&V4 zM&WR8+wErQ7UIc6LY~Q9x%Sn*Tn>>P`^t&idaOEnOd(Ufw#>NoR^1QdhJ8s`h^|R_ zXX`c5*O~Xdvh%q;7L!_!ohf$NfEBmCde|#uVZvEo>OfEq%+Ns7&_f$OR9xsihRpBb z+cjk8LyDm@U{YN>+r46?nn{7Gh(;WhFw6GAxtcKD+YWV?uge>;+q#Xx4!GpRkVZYu zzsF}1)7$?%s9g9CH=Zs+B%M_)+~*j3L0&Q9u7!|+T`^O{xE6qvAP?XWv9_MrZKdo& z%IyU)$Q95AB4!#hT!_dA>4e@zjOBD*Y=XjtMm)V|+IXzjuM;(l+8aA5#Kaz_$rR6! zj>#&^DidYD$nUY(D$mH`9eb|dtV0b{S>H6FBfq>t5`;OxA4Nn{J(+XihF(stSche7$es&~N$epi&PDM_N`As;*9D^L==2Q7Z2zD+CiU(|+-kL*VG+&9!Yb3LgPy?A zm7Z&^qRG_JIxK7-FBzZI3Q<;{`DIxtc48k> zc|0dmX;Z=W$+)qE)~`yn6MdoJ4co;%!`ddy+FV538Y)j(vg}5*k(WK)KWZ3WaOG!8 z!syGn=s{H$odtpqFrT#JGM*utN7B((abXnpDM6w56nhw}OY}0TiTG1#f*VFZr+^-g zbP10`$LPq_;PvrA1XXlyx2uM^mrjTzX}w{yuLo-cOClE8MMk47T25G8M!9Z5ypOSV zAJUBGEg5L2fY)ZGJb^E34R2zJ?}Vf>{~gB!8=5Z) z9y$>5c)=;o0HeHHSuE4U)#vG&KF|I%-cF6f$~pdYJWk_dD}iOA>iA$O$+4%@>JU08 zS`ep)$XLPJ+n0_i@PkF#ri6T8?ZeAot$6JIYHm&P6EB=BiaNY|aA$W0I+nz*zkz_z zkEru!tj!QUffq%)8y0y`T&`fuus-1p>=^hnBiBqD^hXrPs`PY9tU3m0np~rISY09> z`P3s=-kt_cYcxWd{de@}TwSqg*xVhp;E9zCsnXo6z z?f&Sv^U7n4`xr=mXle94HzOdN!2kB~4=%)u&N!+2;z6UYKUDqi-s6AZ!haB;@&B`? z_TRX0%@suz^TRdCb?!vNJYPY8L_}&07uySH9%W^Tc&1pia6y1q#?*Drf}GjGbPjBS zbOPcUY#*$3sL2x4v_i*Y=N7E$mR}J%|GUI(>WEr+28+V z%v5{#e!UF*6~G&%;l*q*$V?&r$Pp^sE^i-0$+RH3ERUUdQ0>rAq2(2QAbG}$y{de( z>{qD~GGuOk559Y@%$?N^1ApVL_a704>8OD%8Y%8B;FCt%AoPu8*D1 zLB5X>b}Syz81pn;xnB}%0FnwazlWfUV)Z-~rZg6~b z6!9J$EcE&sEbzcy?CI~=boWA&eeIa%z(7SE^qgVLz??1Vbc1*aRvc%Mri)AJaAG!p z$X!_9Ds;Zz)f+;%s&dRcJt2==P{^j3bf0M=nJd&xwUGlUFn?H=2W(*2I2Gdu zv!gYCwM10aeus)`RIZSrCK=&oKaO_Ry~D1B5!y0R=%!i2*KfXGYX&gNv_u+n9wiR5 z*e$Zjju&ODRW3phN925%S(jL+bCHv6rZtc?!*`1TyYXT6%Ju=|X;6D@lq$8T zW{Y|e39ioPez(pBH%k)HzFITXHvnD6hw^lIoUMA;qAJ^CU?top1fo@s7xT13Fvn1H z6JWa-6+FJF#x>~+A;D~;VDs26>^oH0EI`IYT2iagy23?nyJ==i{g4%HrAf1-*v zK1)~@&(KkwR7TL}L(A@C_S0G;-GMDy=MJn2$FP5s<%wC)4jC5PXoxrQBFZ_k0P{{s@sz+gX`-!=T8rcB(=7vW}^K6oLWMmp(rwDh}b zwaGGd>yEy6fHv%jM$yJXo5oMAQ>c9j`**}F?MCry;T@47@r?&sKHgVe$MCqk#Z_3S z1GZI~nOEN*P~+UaFGnj{{Jo@16`(qVNtbU>O0Hf57-P>x8Jikp=`s8xWs^dAJ9lCQ z)GFm+=OV%AMVqVATtN@|vp61VVAHRn87}%PC^RAzJ%JngmZTasWBAWsoAqBU+8L8u z4A&Pe?fmTm0?mK-BL9t+{y7o(7jm+RpOhL9KnY#E&qu^}B6=K_dB}*VlSEiC9fn)+V=J;OnN)Ta5v66ic1rG+dGAJ1 z1%Zb_+!$=tQ~lxQrzv3x#CPb?CekEkA}0MYSgx$Jdd}q8+R=ma$|&1a#)TQ=l$1tQ z=tL9&_^vJ)Pk}EDO-va`UCT1m#Uty1{v^A3P~83_#v^ozH}6*9mIjIr;t3Uv%@VeW zGL6(CwCUp)Jq%G0bIG%?{_*Y#5IHf*5M@wPo6A{$Um++Co$wLC=J1aoG93&T7Ho}P z=mGEPP7GbvoG!uD$k(H3A$Z))+i{Hy?QHdk>3xSBXR0j!11O^mEe9RHmw!pvzv?Ua~2_l2Yh~_!s1qS`|0~0)YsbHSz8!mG)WiJE| z2f($6TQtt6L_f~ApQYQKSb=`053LgrQq7G@98#igV>y#i==-nEjQ!XNu9 z~;mE+gtj4IDDNQJ~JVk5Ux6&LCSFL!y=>79kE9=V}J7tD==Ga+IW zX)r7>VZ9dY=V&}DR))xUoV!u(Z|%3ciQi_2jl}3=$Agc(`RPb z8kEBpvY>1FGQ9W$n>Cq=DIpski};nE)`p3IUw1Oz0|wxll^)4dq3;CCY@RyJgFgc# zKouFh!`?Xuo{IMz^xi-h=StCis_M7yq$u) z?XHvw*HP0VgR+KR6wI)jEMX|ssqYvSf*_3W8zVTQzD?3>H!#>InzpSO)@SC8q*ii- z%%h}_#0{4JG;Jm`4zg};BPTGkYamx$Xo#O~lBirRY)q=5M45n{GCfV7h9qwyu1NxOMoP4)jjZMxmT|IQQh0U7C$EbnMN<3)Kk?fFHYq$d|ICu>KbY_hO zTZM+uKHe(cIZfEqyzyYSUBZa8;Fcut-GN!HSA9ius`ltNebF46ZX_BbZNU}}ZOm{M2&nANL9@0qvih15(|`S~z}m&h!u4x~(%MAO$jHRWNfuxWF#B)E&g3ghSQ9|> z(MFaLQj)NE0lowyjvg8z0#m6FIuKE9lDO~Glg}nSb7`~^&#(Lw{}GVOS>U)m8bF}x zVjbXljBm34Cs-yM6TVusr+3kYFjr28STT3g056y3cH5Tmge~ASxBj z%|yb>$eF;WgrcOZf569sDZOVwoo%8>XO>XQOX1OyN9I-SQgrm;U;+#3OI(zrWyow3 zk==|{lt2xrQ%FIXOTejR>;wv(Pb8u8}BUpx?yd(Abh6? zsoO3VYWkeLnF43&@*#MQ9-i-d0t*xN-UEyNKeyNMHw|A(k(_6QKO=nKMCxD(W(Yop zsRQ)QeL4X3Lxp^L%wzi2-WVSsf61dqliPUM7srDB?Wm6Lzn0&{*}|IsKQW;02(Y&| zaTKv|`U(pSzuvR6Rduu$wzK_W-Y-7>7s?G$)U}&uK;<>vU}^^ns@Z!p+9?St1s)dG zK%y6xkPyyS1$~&6v{kl?Md6gwM|>mt6Upm>oa8RLD^8T{0?HC!Z>;(Bob7el(DV6x zi`I)$&E&ngwFS@bi4^xFLAn`=fzTC;aimE^!cMI2n@Vo%Ae-ne`RF((&5y6xsjjAZ zVguVoQ?Z9uk$2ON;ersE%PU*xGO@T*;j1BO5#TuZKEf(mB7|g7pcEA=nYJ{s3vlbg zd4-DUlD{*6o%Gc^N!Nptgay>j6E5;3psI+C3Q!1ZIbeCubW%w4pq9)MSDyB{HLm|k zxv-{$$A*pS@csolri$Ge<4VZ}e~78JOL-EVyrbxKra^d{?|NnPp86!q>t<&IP07?Z z^>~IK^k#OEKgRH+LjllZXk7iA>2cfH6+(e&9ku5poo~6y{GC5>(bRK7hwjiurqAiZ zg*DmtgY}v83IjE&AbiWgMyFbaRUPZ{lYiz$U^&Zt2YjG<%m((&_JUbZcfJ22(>bi5 z!J?<7AySj0JZ&<-qXX;mcV!f~>G=sB0KnjWca4}vrtunD^1TrpfeS^4dvFr!65knK zZh`d;*VOkPs4*-9kL>$GP0`(M!j~B;#x?Ba~&s6CopvO86oM?-? zOw#dIRc;6A6T?B`Qp%^<U5 z19x(ywSH$_N+Io!6;e?`tWaM$`=Db!gzx|lQ${DG!zb1Zl&|{kX0y6xvO1o z220r<-oaS^^R2pEyY;=Qllqpmue|5yI~D|iI!IGt@iod{Opz@*ml^w2bNs)p`M(Io z|E;;m*Xpjd9l)4G#KaWfV(t8YUn@A;nK^#xgv=LtnArX|vWQVuw3}B${h+frU2>9^ z!l6)!Uo4`5k`<<;E(ido7M6lKTgWezNLq>U*=uz&s=cc$1%>VrAeOoUtA|T6gO4>UNqsdK=NF*8|~*sl&wI=x9-EGiq*aqV!(VVXA57 zw9*o6Ir8Lj1npUXvlevtn(_+^X5rzdR>#(}4YcB9O50q97%rW2me5_L=%ffYPUSRc z!vv?Kv>dH994Qi>U(a<0KF6NH5b16enCp+mw^Hb3Xs1^tThFpz!3QuN#}KBbww`(h z7GO)1olDqy6?T$()R7y%NYx*B0k_2IBiZ14&8|JPFxeMF{vW>HF-Vi3+ZOI=+qP}n zw(+!WcTd~4ZJX1!ZM&y!+uyt=&i!+~d(V%GjH;-NsEEv6nS1TERt|RHh!0>W4+4pp z1-*EzAM~i`+1f(VEHI8So`S`akPfPTfq*`l{Fz`hS%k#JS0cjT2mS0#QLGf=J?1`he3W*;m4)ce8*WFq1sdP=~$5RlH1EdWm|~dCvKOi4*I_96{^95p#B<(n!d?B z=o`0{t+&OMwKcxiBECznJcfH!fL(z3OvmxP#oWd48|mMjpE||zdiTBdWelj8&Qosv zZFp@&UgXuvJw5y=q6*28AtxZzo-UUpkRW%ne+Ylf!V-0+uQXBW=5S1o#6LXNtY5!I z%Rkz#(S8Pjz*P7bqB6L|M#Er{|QLae-Y{KA>`^} z@lPjeX>90X|34S-7}ZVXe{wEei1<{*e8T-Nbj8JmD4iwcE+Hg_zhkPVm#=@b$;)h6 z<<6y`nPa`f3I6`!28d@kdM{uJOgM%`EvlQ5B2bL)Sl=|y@YB3KeOzz=9cUW3clPAU z^sYc}xf9{4Oj?L5MOlYxR{+>w=vJjvbyO5}ptT(o6dR|ygO$)nVCvNGnq(6;bHlBd zl?w-|plD8spjDF03g5ip;W3Z z><0{BCq!Dw;h5~#1BuQilq*TwEu)qy50@+BE4bX28+7erX{BD4H)N+7U`AVEuREE8 z;X?~fyhF-x_sRfHIj~6f(+^@H)D=ngP;mwJjxhQUbUdzk8f94Ab%59-eRIq?ZKrwD z(BFI=)xrUlgu(b|hAysqK<}8bslmNNeD=#JW*}^~Nrswn^xw*nL@Tx!49bfJecV&KC2G4q5a!NSv)06A_5N3Y?veAz;Gv+@U3R% z)~UA8-0LvVE{}8LVDOHzp~2twReqf}ODIyXMM6=W>kL|OHcx9P%+aJGYi_Om)b!xe zF40Vntn0+VP>o<$AtP&JANjXBn7$}C@{+@3I@cqlwR2MdwGhVPxlTIcRVu@Ho-wO` z_~Or~IMG)A_`6-p)KPS@cT9mu9RGA>dVh5wY$NM9-^c@N=hcNaw4ITjm;iWSP^ZX| z)_XpaI61<+La+U&&%2a z0za$)-wZP@mwSELo#3!PGTt$uy0C(nTT@9NX*r3Ctw6J~7A(m#8fE)0RBd`TdKfAT zCf@$MAxjP`O(u9s@c0Fd@|}UQ6qp)O5Q5DPCeE6mSIh|Rj{$cAVIWsA=xPKVKxdhg zLzPZ`3CS+KIO;T}0Ip!fAUaNU>++ZJZRk@I(h<)RsJUhZ&Ru9*!4Ptn;gX^~4E8W^TSR&~3BAZc#HquXn)OW|TJ`CTahk+{qe`5+ixON^zA9IFd8)kc%*!AiLu z>`SFoZ5bW-%7}xZ>gpJcx_hpF$2l+533{gW{a7ce^B9sIdmLrI0)4yivZ^(Vh@-1q zFT!NQK$Iz^xu%|EOK=n>ug;(7J4OnS$;yWmq>A;hsD_0oAbLYhW^1Vdt9>;(JIYjf zdb+&f&D4@4AS?!*XpH>8egQvSVX`36jMd>$+RgI|pEg))^djhGSo&#lhS~9%NuWfX zDDH;3T*GzRT@5=7ibO>N-6_XPBYxno@mD_3I#rDD?iADxX`! zh*v8^i*JEMzyN#bGEBz7;UYXki*Xr(9xXax(_1qVW=Ml)kSuvK$coq2A(5ZGhs_pF z$*w}FbN6+QDseuB9=fdp_MTs)nQf!2SlROQ!gBJBCXD&@-VurqHj0wm@LWX-TDmS= z71M__vAok|@!qgi#H&H%Vg-((ZfxPAL8AI{x|VV!9)ZE}_l>iWk8UPTGHs*?u7RfP z5MC&=c6X;XlUzrz5q?(!eO@~* zoh2I*%J7dF!!_!vXoSIn5o|wj1#_>K*&CIn{qSaRc&iFVxt*^20ngCL;QonIS>I5^ zMw8HXm>W0PGd*}Ko)f|~dDd%;Wu_RWI_d;&2g6R3S63Uzjd7dn%Svu-OKpx*o|N>F zZg=-~qLb~VRLpv`k zWSdfHh@?dp=s_X`{yxOlxE$4iuyS;Z-x!*E6eqmEm*j2bE@=ZI0YZ5%Yj29!5+J$4h{s($nakA`xgbO8w zi=*r}PWz#lTL_DSAu1?f%-2OjD}NHXp4pXOsCW;DS@BC3h-q4_l`<))8WgzkdXg3! zs1WMt32kS2E#L0p_|x+x**TFV=gn`m9BWlzF{b%6j-odf4{7a4y4Uaef@YaeuPhU8 zHBvRqN^;$Jizy+ z=zW{E5<>2gp$pH{M@S*!sJVQU)b*J5*bX4h>5VJve#Q6ga}cQ&iL#=(u+KroWrxa%8&~p{WEUF0il=db;-$=A;&9M{Rq`ouZ5m%BHT6%st%saGsD6)fQgLN}x@d3q>FC;=f%O3Cyg=Ke@Gh`XW za@RajqOE9UB6eE=zhG%|dYS)IW)&y&Id2n7r)6p_)vlRP7NJL(x4UbhlcFXWT8?K=%s7;z?Vjts?y2+r|uk8Wt(DM*73^W%pAkZa1Jd zNoE)8FvQA>Z`eR5Z@Ig6kS5?0h;`Y&OL2D&xnnAUzQz{YSdh0k zB3exx%A2TyI)M*EM6htrxSlep!Kk(P(VP`$p0G~f$smld6W1r_Z+o?=IB@^weq>5VYsYZZR@` z&XJFxd5{|KPZmVOSxc@^%71C@;z}}WhbF9p!%yLj3j%YOlPL5s>7I3vj25 z@xmf=*z%Wb4;Va6SDk9cv|r*lhZ`(y_*M@>q;wrn)oQx%B(2A$9(74>;$zmQ!4fN; z>XurIk-7@wZys<+7XL@0Fhe-f%*=(weaQEdR9Eh6>Kl-EcI({qoZqyzziGwpg-GM#251sK_ z=3|kitS!j%;fpc@oWn65SEL73^N&t>Ix37xgs= zYG%eQDJc|rqHFia0!_sm7`@lvcv)gfy(+KXA@E{3t1DaZ$DijWAcA)E0@X?2ziJ{v z&KOYZ|DdkM{}t+@{@*6ge}m%xfjIxi%qh`=^2Rwz@w0cCvZ&Tc#UmCDbVwABrON^x zEBK43FO@weA8s7zggCOWhMvGGE`baZ62cC)VHyy!5Zbt%ieH+XN|OLbAFPZWyC6)p z4P3%8sq9HdS3=ih^0OOlqTPbKuzQ?lBEI{w^ReUO{V?@`ARsL|S*%yOS=Z%sF)>-y z(LAQdhgAcuF6LQjRYfdbD1g4o%tV4EiK&ElLB&^VZHbrV1K>tHTO{#XTo>)2UMm`2 z^t4s;vnMQgf-njU-RVBRw0P0-m#d-u`(kq7NL&2T)TjI_@iKuPAK-@oH(J8?%(e!0Ir$yG32@CGUPn5w4)+9@8c&pGx z+K3GKESI4*`tYlmMHt@br;jBWTei&(a=iYslc^c#RU3Q&sYp zSG){)V<(g7+8W!Wxeb5zJb4XE{I|&Y4UrFWr%LHkdQ;~XU zgy^dH-Z3lmY+0G~?DrC_S4@=>0oM8Isw%g(id10gWkoz2Q%7W$bFk@mIzTCcIB(K8 zc<5h&ZzCdT=9n-D>&a8vl+=ZF*`uTvQviG_bLde*k>{^)&0o*b05x$MO3gVLUx`xZ z43j+>!u?XV)Yp@MmG%Y`+COH2?nQcMrQ%k~6#O%PeD_WvFO~Kct za4XoCM_X!c5vhRkIdV=xUB3xI2NNStK*8_Zl!cFjOvp-AY=D;5{uXj}GV{LK1~IE2 z|KffUiBaStRr;10R~K2VVtf{TzM7FaPm;Y(zQjILn+tIPSrJh&EMf6evaBKIvi42-WYU9Vhj~3< zZSM-B;E`g_o8_XTM9IzEL=9Lb^SPhe(f(-`Yh=X6O7+6ALXnTcUFpI>ekl6v)ZQeNCg2 z^H|{SKXHU*%nBQ@I3It0m^h+6tvI@FS=MYS$ZpBaG7j#V@P2ZuYySbp@hA# ze(kc;P4i_-_UDP?%<6>%tTRih6VBgScKU^BV6Aoeg6Uh(W^#J^V$Xo^4#Ekp ztqQVK^g9gKMTHvV7nb64UU7p~!B?>Y0oFH5T7#BSW#YfSB@5PtE~#SCCg3p^o=NkMk$<8- z6PT*yIKGrvne7+y3}_!AC8NNeI?iTY(&nakN>>U-zT0wzZf-RuyZk^X9H-DT_*wk= z;&0}6LsGtfVa1q)CEUPlx#(ED@-?H<1_FrHU#z5^P3lEB|qsxEyn%FOpjx z3S?~gvoXy~L(Q{Jh6*i~=f%9kM1>RGjBzQh_SaIDfSU_9!<>*Pm>l)cJD@wlyxpBV z4Fmhc2q=R_wHCEK69<*wG%}mgD1=FHi4h!98B-*vMu4ZGW~%IrYSLGU{^TuseqVgV zLP<%wirIL`VLyJv9XG_p8w@Q4HzNt-o;U@Au{7%Ji;53!7V8Rv0^Lu^Vf*sL>R(;c zQG_ZuFl)Mh-xEIkGu}?_(HwkB2jS;HdPLSxVU&Jxy9*XRG~^HY(f0g8Q}iqnVmgjI zfd=``2&8GsycjR?M%(zMjn;tn9agcq;&rR!Hp z$B*gzHsQ~aXw8c|a(L^LW(|`yGc!qOnV(ZjU_Q-4z1&0;jG&vAKuNG=F|H?@m5^N@ zq{E!1n;)kNTJ>|Hb2ODt-7U~-MOIFo%9I)_@7fnX+eMMNh>)V$IXesJpBn|uo8f~#aOFytCT zf9&%MCLf8mp4kwHTcojWmM3LU=#|{3L>E}SKwOd?%{HogCZ_Z1BSA}P#O(%H$;z7XyJ^sjGX;j5 zrzp>|Ud;*&VAU3x#f{CKwY7Vc{%TKKqmB@oTHA9;>?!nvMA;8+Jh=cambHz#J18x~ zs!dF>$*AnsQ{{82r5Aw&^7eRCdvcgyxH?*DV5(I$qXh^zS>us*I66_MbL8y4d3ULj z{S(ipo+T3Ag!+5`NU2sc+@*m{_X|&p#O-SAqF&g_n7ObB82~$p%fXA5GLHMC+#qqL zdt`sJC&6C2)=juQ_!NeD>U8lDVpAOkW*khf7MCcs$A(wiIl#B9HM%~GtQ^}yBPjT@ z+E=|A!Z?A(rwzZ;T}o6pOVqHzTr*i;Wrc%&36kc@jXq~+w8kVrs;%=IFdACoLAcCAmhFNpbP8;s`zG|HC2Gv?I~w4ITy=g$`0qMQdkijLSOtX6xW%Z9Nw<;M- zMN`c7=$QxN00DiSjbVt9Mi6-pjv*j(_8PyV-il8Q-&TwBwH1gz1uoxs6~uU}PrgWB zIAE_I-a1EqlIaGQNbcp@iI8W1sm9fBBNOk(k&iLBe%MCo#?xI$%ZmGA?=)M9D=0t7 zc)Q0LnI)kCy{`jCGy9lYX%mUsDWwsY`;jE(;Us@gmWPqjmXL+Hu#^;k%eT>{nMtzj zsV`Iy6leTA8-PndszF;N^X@CJrTw5IIm!GPeu)H2#FQitR{1p;MasQVAG3*+=9FYK zw*k!HT(YQorfQj+1*mCV458(T5=fH`um$gS38hw(OqVMyunQ;rW5aPbF##A3fGH6h z@W)i9Uff?qz`YbK4c}JzQpuxuE3pcQO)%xBRZp{zJ^-*|oryTxJ-rR+MXJ)!f=+pp z10H|DdGd2exhi+hftcYbM0_}C0ZI-2vh+$fU1acsB-YXid7O|=9L!3e@$H*6?G*Zp z%qFB(sgl=FcC=E4CYGp4CN>=M8#5r!RU!u+FJVlH6=gI5xHVD&k;Ta*M28BsxfMV~ zLz+@6TxnfLhF@5=yQo^1&S}cmTN@m!7*c6z;}~*!hNBjuE>NLVl2EwN!F+)0$R1S! zR|lF%n!9fkZ@gPW|x|B={V6x3`=jS*$Pu0+5OWf?wnIy>Y1MbbGSncpKO0qE(qO=ts z!~@&!N`10S593pVQu4FzpOh!tvg}p%zCU(aV5=~K#bKi zHdJ1>tQSrhW%KOky;iW+O_n;`l9~omqM%sdxdLtI`TrJzN6BQz+7xOl*rM>xVI2~# z)7FJ^Dc{DC<%~VS?@WXzuOG$YPLC;>#vUJ^MmtbSL`_yXtNKa$Hk+l-c!aC7gn(Cg ze?YPYZ(2Jw{SF6MiO5(%_pTo7j@&DHNW`|lD`~{iH+_eSTS&OC*2WTT*a`?|9w1dh zh1nh@$a}T#WE5$7Od~NvSEU)T(W$p$s5fe^GpG+7fdJ9=enRT9$wEk+ZaB>G3$KQO zgq?-rZZnIv!p#>Ty~}c*Lb_jxJg$eGM*XwHUwuQ|o^}b3^T6Bxx{!?va8aC@-xK*H ztJBFvFfsSWu89%@b^l3-B~O!CXs)I6Y}y#0C0U0R0WG zybjroj$io0j}3%P7zADXOwHwafT#uu*zfM!oD$6aJx7+WL%t-@6^rD_a_M?S^>c;z zMK580bZXo1f*L$CuMeM4Mp!;P@}b~$cd(s5*q~FP+NHSq;nw3fbWyH)i2)-;gQl{S zZO!T}A}fC}vUdskGSq&{`oxt~0i?0xhr6I47_tBc`fqaSrMOzR4>0H^;A zF)hX1nfHs)%Zb-(YGX;=#2R6C{BG;k=?FfP?9{_uFLri~-~AJ;jw({4MU7e*d)?P@ zXX*GkNY9ItFjhwgAIWq7Y!ksbMzfqpG)IrqKx9q{zu%Mdl+{Dis#p9q`02pr1LG8R z@As?eG!>IoROgS!@J*to<27coFc1zpkh?w=)h9CbYe%^Q!Ui46Y*HO0mr% zEff-*$ndMNw}H2a5@BsGj5oFfd!T(F&0$<{GO!Qdd?McKkorh=5{EIjDTHU`So>8V zBA-fqVLb2;u7UhDV1xMI?y>fe3~4urv3%PX)lDw+HYa;HFkaLqi4c~VtCm&Ca+9C~ zge+67hp#R9`+Euq59WhHX&7~RlXn=--m8$iZ~~1C8cv^2(qO#X0?vl91gzUKBeR1J z^p4!!&7)3#@@X&2aF2-)1Ffcc^F8r|RtdL2X%HgN&XU-KH2SLCbpw?J5xJ*!F-ypZ zMG%AJ!Pr&}`LW?E!K~=(NJxuSVTRCGJ$2a*Ao=uUDSys!OFYu!Vs2IT;xQ6EubLIl z+?+nMGeQQhh~??0!s4iQ#gm3!BpMpnY?04kK375e((Uc7B3RMj;wE?BCoQGu=UlZt!EZ1Q*auI)dj3Jj{Ujgt zW5hd~-HWBLI_3HuO) zNrb^XzPsTIb=*a69wAAA3J6AAZZ1VsYbIG}a`=d6?PjM)3EPaDpW2YP$|GrBX{q*! z$KBHNif)OKMBCFP5>!1d=DK>8u+Upm-{hj5o|Wn$vh1&K!lVfDB&47lw$tJ?d5|=B z^(_9=(1T3Fte)z^>|3**n}mIX;mMN5v2F#l(q*CvU{Ga`@VMp#%rQkDBy7kYbmb-q z<5!4iuB#Q_lLZ8}h|hPODI^U6`gzLJre9u3k3c#%86IKI*^H-@I48Bi*@avYm4v!n0+v zWu{M{&F8#p9cx+gF0yTB_<2QUrjMPo9*7^-uP#~gGW~y3nfPAoV%amgr>PSyVAd@l)}8#X zR5zV6t*uKJZL}?NYvPVK6J0v4iVpwiN|>+t3aYiZSp;m0!(1`bHO}TEtWR1tY%BPB z(W!0DmXbZAsT$iC13p4f>u*ZAy@JoLAkJhzFf1#4;#1deO8#8d&89}en&z!W&A3++^1(;>0SB1*54d@y&9Pn;^IAf3GiXbfT`_>{R+Xv; zQvgL>+0#8-laO!j#-WB~(I>l0NCMt_;@Gp_f0#^c)t?&#Xh1-7RR0@zPyBz!U#0Av zT?}n({(p?p7!4S2ZBw)#KdCG)uPnZe+U|0{BW!m)9 zi_9$F?m<`2!`JNFv+w8MK_K)qJ^aO@7-Ig>cM4-r0bi=>?B_2mFNJ}aE3<+QCzRr*NA!QjHw# z`1OsvcoD0?%jq{*7b!l|L1+Tw0TTAM4XMq7*ntc-Ived>Sj_ZtS|uVdpfg1_I9knY z2{GM_j5sDC7(W&}#s{jqbybqJWyn?{PW*&cQIU|*v8YGOKKlGl@?c#TCnmnAkAzV- zmK={|1G90zz=YUvC}+fMqts0d4vgA%t6Jhjv?d;(Z}(Ep8fTZfHA9``fdUHkA+z3+ zhh{ohP%Bj?T~{i0sYCQ}uC#5BwN`skI7`|c%kqkyWIQ;!ysvA8H`b-t()n6>GJj6xlYDu~8qX{AFo$Cm3d|XFL=4uvc?Keb zzb0ZmMoXca6Mob>JqkNuoP>B2Z>D`Q(TvrG6m`j}-1rGP!g|qoL=$FVQYxJQjFn33lODt3Wb1j8VR zlR++vIT6^DtYxAv_hxupbLLN3e0%A%a+hWTKDV3!Fjr^cWJ{scsAdfhpI)`Bms^M6 zQG$waKgFr=c|p9Piug=fcJvZ1ThMnNhQvBAg-8~b1?6wL*WyqXhtj^g(Ke}mEfZVM zJuLNTUVh#WsE*a6uqiz`b#9ZYg3+2%=C(6AvZGc=u&<6??!slB1a9K)=VL zY9EL^mfyKnD zSJyYBc_>G;5RRnrNgzJz#Rkn3S1`mZgO`(r5;Hw6MveN(URf_XS-r58Cn80K)ArH4 z#Rrd~LG1W&@ttw85cjp8xV&>$b%nSXH_*W}7Ch2pg$$c0BdEo-HWRTZcxngIBJad> z;C>b{jIXjb_9Jis?NZJsdm^EG}e*pR&DAy0EaSGi3XWTa(>C%tz1n$u?5Fb z1qtl?;_yjYo)(gB^iQq?=jusF%kywm?CJP~zEHi0NbZ);$(H$w(Hy@{i>$wcVRD_X|w-~(0Z9BJyh zhNh;+eQ9BEIs;tPz%jSVnfCP!3L&9YtEP;svoj_bNzeGSQIAjd zBss@A;)R^WAu-37RQrM%{DfBNRx>v!G31Z}8-El9IOJlb_MSoMu2}GDYycNaf>uny z+8xykD-7ONCM!APry_Lw6-yT>5!tR}W;W`C)1>pxSs5o1z#j7%m=&=7O4hz+Lsqm` z*>{+xsabZPr&X=}G@obTb{nPTkccJX8w3CG7X+1+t{JcMabv~UNv+G?txRqXib~c^Mo}`q{$`;EBNJ;#F*{gvS12kV?AZ%O0SFB$^ zn+}!HbmEj}w{Vq(G)OGAzH}R~kS^;(-s&=ectz8vN!_)Yl$$U@HNTI-pV`LSj7Opu zTZ5zZ)-S_{GcEQPIQXLQ#oMS`HPu{`SQiAZ)m1at*Hy%3xma|>o`h%E%8BEbi9p0r zVjcsh<{NBKQ4eKlXU|}@XJ#@uQw*$4BxKn6#W~I4T<^f99~(=}a`&3(ur8R9t+|AQ zWkQx7l}wa48-jO@ft2h+7qn%SJtL%~890FG0s5g*kNbL3I&@brh&f6)TlM`K^(bhr zJWM6N6x3flOw$@|C@kPi7yP&SP?bzP-E|HSXQXG>7gk|R9BTj`e=4de9C6+H7H7n# z#GJeVs1mtHhLDmVO?LkYRQc`DVOJ_vdl8VUihO-j#t=0T3%Fc1f9F73ufJz*adn*p zc%&vi(4NqHu^R>sAT_0EDjVR8bc%wTz#$;%NU-kbDyL_dg0%TFafZwZ?5KZpcuaO54Z9hX zD$u>q!-9`U6-D`E#`W~fIfiIF5_m6{fvM)b1NG3xf4Auw;Go~Fu7cth#DlUn{@~yu z=B;RT*dp?bO}o%4x7k9v{r=Y@^YQ^UUm(Qmliw8brO^=NP+UOohLYiaEB3^DB56&V zK?4jV61B|1Uj_5fBKW;8LdwOFZKWp)g{B%7g1~DgO&N& z#lisxf?R~Z@?3E$Mms$$JK8oe@X`5m98V*aV6Ua}8Xs2#A!{x?IP|N(%nxsH?^c{& z@vY&R1QmQs83BW28qAmJfS7MYi=h(YK??@EhjL-t*5W!p z^gYX!Q6-vBqcv~ruw@oMaU&qp0Fb(dbVzm5xJN%0o_^@fWq$oa3X?9s%+b)x4w-q5Koe(@j6Ez7V@~NRFvd zfBH~)U5!ix3isg`6be__wBJp=1@yfsCMw1C@y+9WYD9_C%{Q~7^0AF2KFryfLlUP# zwrtJEcH)jm48!6tUcxiurAMaiD04C&tPe6DI0#aoqz#Bt0_7_*X*TsF7u*zv(iEfA z;$@?XVu~oX#1YXtceQL{dSneL&*nDug^OW$DSLF0M1Im|sSX8R26&)<0Fbh^*l6!5wfSu8MpMoh=2l z^^0Sr$UpZp*9oqa23fcCfm7`ya2<4wzJ`Axt7e4jJrRFVf?nY~2&tRL* zd;6_njcz01c>$IvN=?K}9ie%Z(BO@JG2J}fT#BJQ+f5LFSgup7i!xWRKw6)iITjZU z%l6hPZia>R!`aZjwCp}I zg)%20;}f+&@t;(%5;RHL>K_&7MH^S+7<|(SZH!u zznW|jz$uA`P9@ZWtJgv$EFp>)K&Gt+4C6#*khZQXS*S~6N%JDT$r`aJDs9|uXWdbg zBwho$phWx}x!qy8&}6y5Vr$G{yGSE*r$^r{}pw zVTZKvikRZ`J_IJrjc=X1uw?estdwm&bEahku&D04HD+0Bm~q#YGS6gp!KLf$A{%Qd z&&yX@Hp>~(wU{|(#U&Bf92+1i&Q*-S+=y=3pSZy$#8Uc$#7oiJUuO{cE6=tsPhwPe| zxQpK>`Dbka`V)$}e6_OXKLB%i76~4N*zA?X+PrhH<&)}prET;kel24kW%+9))G^JI zsq7L{P}^#QsZViX%KgxBvEugr>ZmFqe^oAg?{EI=&_O#e)F3V#rc z8$4}0Zr19qd3tE4#$3_f=Bbx9oV6VO!d3(R===i-7p=Vj`520w0D3W6lQfY48}!D* z&)lZMG;~er2qBoI2gsX+Ts-hnpS~NYRDtPd^FPzn!^&yxRy#CSz(b&E*tL|jIkq|l zf%>)7Dtu>jCf`-7R#*GhGn4FkYf;B$+9IxmqH|lf6$4irg{0ept__%)V*R_OK=T06 zyT_m-o@Kp6U{l5h>W1hGq*X#8*y@<;vsOFqEjTQXFEotR+{3}ODDnj;o0@!bB5x=N z394FojuGOtVKBlVRLtHp%EJv_G5q=AgF)SKyRN5=cGBjDWv4LDn$IL`*=~J7u&Dy5 zrMc83y+w^F&{?X(KOOAl-sWZDb{9X9#jrQtmrEXD?;h-}SYT7yM(X_6qksM=K_a;Z z3u0qT0TtaNvDER_8x*rxXw&C^|h{P1qxK|@pS7vdlZ#P z7PdB7MmC2}%sdzAxt>;WM1s0??`1983O4nFK|hVAbHcZ3x{PzytQLkCVk7hA!Lo` zEJH?4qw|}WH{dc4z%aB=0XqsFW?^p=X}4xnCJXK%c#ItOSjdSO`UXJyuc8bh^Cf}8 z@Ht|vXd^6{Fgai8*tmyRGmD_s_nv~r^Fy7j`Bu`6=G)5H$i7Q7lvQnmea&TGvJp9a|qOrUymZ$6G|Ly z#zOCg++$3iB$!6!>215A4!iryregKuUT344X)jQb3|9qY>c0LO{6Vby05n~VFzd?q zgGZv&FGlkiH*`fTurp>B8v&nSxNz)=5IF$=@rgND4d`!AaaX;_lK~)-U8la_Wa8i?NJC@BURO*sUW)E9oyv3RG^YGfN%BmxzjlT)bp*$<| zX3tt?EAy<&K+bhIuMs-g#=d1}N_?isY)6Ay$mDOKRh z4v1asEGWoAp=srraLW^h&_Uw|6O+r;wns=uwYm=JN4Q!quD8SQRSeEcGh|Eb5Jg8m zOT}u;N|x@aq)=&;wufCc^#)5U^VcZw;d_wwaoh9$p@Xrc{DD6GZUqZ ziC6OT^zSq@-lhbgR8B+e;7_Giv;DK5gn^$bs<6~SUadiosfewWDJu`XsBfOd1|p=q zE>m=zF}!lObA%ePey~gqU8S6h-^J2Y?>7)L2+%8kV}Gp=h`Xm_}rlm)SyUS=`=S7msKu zC|T!gPiI1rWGb1z$Md?0YJQ;%>uPLOXf1Z>N~`~JHJ!^@D5kSXQ4ugnFZ>^`zH8CAiZmp z6Ms|#2gcGsQ{{u7+Nb9sA?U>(0e$5V1|WVwY`Kn)rsnnZ4=1u=7u!4WexZD^IQ1Jk zfF#NLe>W$3m&C^ULjdw+5|)-BSHwpegdyt9NYC{3@QtMfd8GrIWDu`gd0nv-3LpGCh@wgBaG z176tikL!_NXM+Bv#7q^cyn9$XSeZR6#!B4JE@GVH zoobHZN_*RF#@_SVYKkQ_igme-Y5U}cV(hkR#k1c{bQNMji zU7aE`?dHyx=1`kOYZo_8U7?3-7vHOp`Qe%Z*i+FX!s?6huNp0iCEW-Z7E&jRWmUW_ z67j>)Ew!yq)hhG4o?^z}HWH-e=es#xJUhDRc4B51M4~E-l5VZ!&zQq`gWe`?}#b~7w1LH4Xa-UCT5LXkXQWheBa2YJYbyQ zl1pXR%b(KCXMO0OsXgl0P0Og<{(@&z1aokU-Pq`eQq*JYgt8xdFQ6S z6Z3IFSua8W&M#`~*L#r>Jfd6*BzJ?JFdBR#bDv$_0N!_5vnmo@!>vULcDm`MFU823 zpG9pqjqz^FE5zMDoGqhs5OMmC{Y3iVcl>F}5Rs24Y5B^mYQ;1T&ks@pIApHOdrzXF z-SdX}Hf{X;TaSxG_T$0~#RhqKISGKNK47}0*x&nRIPtmdwxc&QT3$8&!3fWu1eZ_P zJveQj^hJL#Sn!*4k`3}(d(aasl&7G0j0-*_2xtAnoX1@9+h zO#c>YQg60Z;o{Bi=3i7S`Ic+ZE>K{(u|#)9y}q*j8uKQ1^>+(BI}m%1v3$=4ojGBc zm+o1*!T&b}-lVvZqIUBc8V}QyFEgm#oyIuC{8WqUNV{Toz`oxhYpP!_p2oHHh5P@iB*NVo~2=GQm+8Yrkm2Xjc_VyHg1c0>+o~@>*Qzo zHVBJS>$$}$_4EniTI;b1WShX<5-p#TPB&!;lP!lBVBbLOOxh6FuYloD%m;n{r|;MU3!q4AVkua~fieeWu2 zQAQ$ue(IklX6+V;F1vCu-&V?I3d42FgWgsb_e^29ol}HYft?{SLf>DrmOp9o!t>I^ zY7fBCk+E8n_|apgM|-;^=#B?6RnFKlN`oR)`e$+;D=yO-(U^jV;rft^G_zl`n7qnM zL z*-Y4Phq+ZI1$j$F-f;`CD#|`-T~OM5Q>x}a>B~Gb3-+9i>Lfr|Ca6S^8g*{*?_5!x zH_N!SoRP=gX1?)q%>QTY!r77e2j9W(I!uAz{T`NdNmPBBUzi2{`XMB^zJGGwFWeA9 z{fk33#*9SO0)DjROug+(M)I-pKA!CX;IY(#gE!UxXVsa)X!UftIN98{pt#4MJHOhY zM$_l}-TJlxY?LS6Nuz1T<44m<4i^8k@D$zuCPrkmz@sdv+{ciyFJG2Zwy&%c7;atIeTdh!a(R^QXnu1Oq1b42*OQFWnyQ zWeQrdvP|w_idy53Wa<{QH^lFmEd+VlJkyiC>6B#s)F;w-{c;aKIm;Kp50HnA-o3lY z9B~F$gJ@yYE#g#X&3ADx&tO+P_@mnQTz9gv30_sTsaGXkfNYXY{$(>*PEN3QL>I!k zp)KibPhrfX3%Z$H6SY`rXGYS~143wZrG2;=FLj50+VM6soI~up_>fU(2Wl@{BRsMi zO%sL3x?2l1cXTF)k&moNsHfQrQ+wu(gBt{sk#CU=UhrvJIncy@tJX5klLjgMn>~h= zg|FR&;@eh|C7`>s_9c~0-{IAPV){l|Ts`i=)AW;d9&KPc3fMeoTS%8@V~D8*h;&(^>yjT84MM}=%#LS7shLAuuj(0VAYoozhWjq z4LEr?wUe2^WGwdTIgWBkDUJa>YP@5d9^Rs$kCXmMRxuF*YMVrn?0NFyPl}>`&dqZb z<5eqR=ZG3>n2{6v6BvJ`YBZeeTtB88TAY(x0a58EWyuf>+^|x8Qa6wA|1Nb_p|nA zWWa}|z8a)--Wj`LqyFk_a3gN2>5{Rl_wbW?#by7&i*^hRknK%jwIH6=dQ8*-_{*x0j^DUfMX0`|K@6C<|1cgZ~D(e5vBFFm;HTZF(!vT8=T$K+|F)x3kqzBV4-=p1V(lzi(s7jdu0>LD#N=$Lk#3HkG!a zIF<7>%B7sRNzJ66KrFV76J<2bdYhxll0y2^_rdG=I%AgW4~)1Nvz=$1UkE^J%BxLo z+lUci`UcU062os*=`-j4IfSQA{w@y|3}Vk?i;&SSdh8n+$iHA#%ERL{;EpXl6u&8@ zzg}?hkEOUOJt?ZL=pWZFJ19mI1@P=$U5*Im1e_8Z${JsM>Ov?nh8Z zP5QvI!{Jy@&BP48%P2{Jr_VgzW;P@7)M9n|lDT|Ep#}7C$&ud&6>C^5ZiwKIg2McPU(4jhM!BD@@L(Gd*Nu$ji(ljZ<{FIeW_1Mmf;76{LU z-ywN~=uNN)Xi6$<12A9y)K%X|(W0p|&>>4OXB?IiYr||WKDOJPxiSe01NSV-h24^L z_>m$;|C+q!Mj**-qQ$L-*++en(g|hw;M!^%_h-iDjFHLo-n3JpB;p?+o2;`*jpvJU zLY^lt)Un4joij^^)O(CKs@7E%*!w>!HA4Q?0}oBJ7Nr8NQ7QmY^4~jvf0-`%waOLn zdNjAPaC0_7c|RVhw)+71NWjRi!y>C+Bl;Z`NiL^zn2*0kmj5gyhCLCxts*cWCdRI| zjsd=sT5BVJc^$GxP~YF$-U{-?kW6r@^vHXB%{CqYzU@1>dzf#3SYedJG-Rm6^RB7s zGM5PR(yKPKR)>?~vpUIeTP7A1sc8-knnJk*9)3t^e%izbdm>Y=W{$wm(cy1RB-19i za#828DMBY+ps#7Y8^6t)=Ea@%Nkt)O6JCx|ybC;Ap}Z@Zw~*}3P>MZLPb4Enxz9Wf zssobT^(R@KuShj8>@!1M7tm|2%-pYYDxz-5`rCbaTCG5{;Uxm z*g=+H1X8{NUvFGzz~wXa%Eo};I;~`37*WrRU&K0dPSB$yk(Z*@K&+mFal^?c zurbqB-+|Kb5|sznT;?Pj!+kgFY1#Dr;_%A(GIQC{3ct|{*Bji%FNa6c-thbpBkA;U zURV!Dr&X{0J}iht#-Qp2=xzuh(fM>zRoiGrYl5ttw2#r34gC41CCOC31m~^UPTK@s z6;A@)7O7_%C)>bnAXerYuAHdE93>j2N}H${zEc6&SbZ|-fiG*-qtGuy-qDelH(|u$ zorf8_T6Zqe#Ub!+e3oSyrskt_HyW_^5lrWt#30l)tHk|j$@YyEkXUOV;6B51L;M@=NIWZXU;GrAa(LGxO%|im%7F<-6N;en0Cr zLH>l*y?pMwt`1*cH~LdBPFY_l;~`N!Clyfr;7w<^X;&(ZiVdF1S5e(+Q%60zgh)s4 zn2yj$+mE=miVERP(g8}G4<85^-5f@qxh2ec?n+$A_`?qN=iyT1?U@t?V6DM~BIlBB z>u~eXm-aE>R0sQy!-I4xtCNi!!qh?R1!kKf6BoH2GG{L4%PAz0{Sh6xpuyI%*~u)s z%rLuFl)uQUCBQAtMyN;%)zFMx4loh7uTfKeB2Xif`lN?2gq6NhWhfz0u5WP9J>=V2 zo{mLtSy&BA!mSzs&CrKWq^y40JF5a&GSXIi2= z{EYb59J4}VwikL4P=>+mc6{($FNE@e=VUwG+KV21;<@lrN`mnz5jYGASyvz7BOG_6(p^eTxD-4O#lROgon;R35=|nj#eHIfJBYPWG>H>`dHKCDZ3`R{-?HO0mE~(5_WYcFmp8sU?wr*UkAQiNDGc6T zA%}GOLXlOWqL?WwfHO8MB#8M8*~Y*gz;1rWWoVSXP&IbKxbQ8+s%4Jnt?kDsq7btI zCDr0PZ)b;B%!lu&CT#RJzm{l{2fq|BcY85`w~3LSK<><@(2EdzFLt9Y_`;WXL6x`0 zDoQ?=?I@Hbr;*VVll1Gmd8*%tiXggMK81a+T(5Gx6;eNb8=uYn z5BG-0g>pP21NPn>$ntBh>`*})Fl|38oC^9Qz>~MAazH%3Q~Qb!ALMf$srexgPZ2@&c~+hxRi1;}+)-06)!#Mq<6GhP z-Q?qmgo${aFBApb5p}$1OJKTClfi8%PpnczyVKkoHw7Ml9e7ikrF0d~UB}i3vizos zXW4DN$SiEV9{faLt5bHy2a>33K%7Td-n5C*N;f&ZqAg#2hIqEb(y<&f4u5BWJ>2^4 z414GosL=Aom#m&=x_v<0-fp1r%oVJ{T-(xnomNJ(Dryv zh?vj+%=II_nV+@NR+(!fZZVM&(W6{6%9cm+o+Z6}KqzLw{(>E86uA1`_K$HqINlb1 zKelh3-jr2I9V?ych`{hta9wQ2c9=MM`2cC{m6^MhlL2{DLv7C^j z$xXBCnDl_;l|bPGMX@*tV)B!c|4oZyftUlP*?$YU9C_eAsuVHJ58?)zpbr30P*C`T z7y#ao`uE-SOG(Pi+`$=e^mle~)pRrdwL5)N;o{gpW21of(QE#U6w%*C~`v-z0QqBML!!5EeYA5IQB0 z^l01c;L6E(iytN!LhL}wfwP7W9PNAkb+)Cst?qg#$n;z41O4&v+8-zPs+XNb-q zIeeBCh#ivnFLUCwfS;p{LC0O7tm+Sf9Jn)~b%uwP{%69;QC)Ok0t%*a5M+=;y8j=v z#!*pp$9@!x;UMIs4~hP#pnfVc!%-D<+wsG@R2+J&%73lK|2G!EQC)O05TCV=&3g)C!lT=czLpZ@Sa%TYuoE?v8T8`V;e$#Zf2_Nj6nvBgh1)2 GZ~q4|mN%#X literal 63721 zcmb5Wb9gP!wgnp7wrv|bwr$&XvSZt}Z6`anZSUAlc9NHKf9JdJ;NJVr`=eI(_pMp0 zy1VAAG3FfAOI`{X1O)&90s;U4K;XLp008~hCjbEC_fbYfS%6kTR+JtXK>nW$ZR+`W ze|#J8f4A@M|F5BpfUJb5h>|j$jOe}0oE!`Zf6fM>CR?!y@zU(cL8NsKk`a z6tx5mAkdjD;J=LcJ;;Aw8p!v#ouk>mUDZF@ zK>yvw%+bKu+T{Nk@LZ;zkYy0HBKw06_IWcMHo*0HKpTsEFZhn5qCHH9j z)|XpN&{`!0a>Vl+PmdQc)Yg4A(AG-z!+@Q#eHr&g<9D?7E)_aEB?s_rx>UE9TUq|? z;(ggJt>9l?C|zoO@5)tu?EV0x_7T17q4fF-q3{yZ^ipUbKcRZ4Qftd!xO(#UGhb2y>?*@{xq%`(-`2T^vc=#< zx!+@4pRdk&*1ht2OWk^Z5IAQ0YTAXLkL{(D*$gENaD)7A%^XXrCchN&z2x+*>o2FwPFjWpeaL=!tzv#JOW#( z$B)Nel<+$bkH1KZv3&-}=SiG~w2sbDbAWarg%5>YbC|}*d9hBjBkR(@tyM0T)FO$# zPtRXukGPnOd)~z=?avu+4Co@wF}1T)-uh5jI<1$HLtyDrVak{gw`mcH@Q-@wg{v^c zRzu}hMKFHV<8w}o*yg6p@Sq%=gkd~;`_VGTS?L@yVu`xuGy+dH6YOwcP6ZE`_0rK% zAx5!FjDuss`FQ3eF|mhrWkjux(Pny^k$u_)dyCSEbAsecHsq#8B3n3kDU(zW5yE|( zgc>sFQywFj5}U*qtF9Y(bi*;>B7WJykcAXF86@)z|0-Vm@jt!EPoLA6>r)?@DIobIZ5Sx zsc@OC{b|3%vaMbyeM|O^UxEYlEMHK4r)V-{r)_yz`w1*xV0|lh-LQOP`OP`Pk1aW( z8DSlGN>Ts|n*xj+%If~+E_BxK)~5T#w6Q1WEKt{!Xtbd`J;`2a>8boRo;7u2M&iOop4qcy<)z023=oghSFV zST;?S;ye+dRQe>ygiJ6HCv4;~3DHtJ({fWeE~$H@mKn@Oh6Z(_sO>01JwH5oA4nvK zr5Sr^g+LC zLt(i&ecdmqsIJGNOSUyUpglvhhrY8lGkzO=0USEKNL%8zHshS>Qziu|`eyWP^5xL4 zRP122_dCJl>hZc~?58w~>`P_s18VoU|7(|Eit0-lZRgLTZKNq5{k zE?V=`7=R&ro(X%LTS*f+#H-mGo_j3dm@F_krAYegDLk6UV{`UKE;{YSsn$ z(yz{v1@p|p!0>g04!eRSrSVb>MQYPr8_MA|MpoGzqyd*$@4j|)cD_%^Hrd>SorF>@ zBX+V<@vEB5PRLGR(uP9&U&5=(HVc?6B58NJT_igiAH*q~Wb`dDZpJSKfy5#Aag4IX zj~uv74EQ_Q_1qaXWI!7Vf@ZrdUhZFE;L&P_Xr8l@GMkhc#=plV0+g(ki>+7fO%?Jb zl+bTy7q{w^pTb{>(Xf2q1BVdq?#f=!geqssXp z4pMu*q;iiHmA*IjOj4`4S&|8@gSw*^{|PT}Aw~}ZXU`6=vZB=GGeMm}V6W46|pU&58~P+?LUs%n@J}CSrICkeng6YJ^M? zS(W?K4nOtoBe4tvBXs@@`i?4G$S2W&;$z8VBSM;Mn9 zxcaEiQ9=vS|bIJ>*tf9AH~m&U%2+Dim<)E=}KORp+cZ^!@wI`h1NVBXu{@%hB2Cq(dXx_aQ9x3mr*fwL5!ZryQqi|KFJuzvP zK1)nrKZ7U+B{1ZmJub?4)Ln^J6k!i0t~VO#=q1{?T)%OV?MN}k5M{}vjyZu#M0_*u z8jwZKJ#Df~1jcLXZL7bnCEhB6IzQZ-GcoQJ!16I*39iazoVGugcKA{lhiHg4Ta2fD zk1Utyc5%QzZ$s3;p0N+N8VX{sd!~l*Ta3|t>lhI&G`sr6L~G5Lul`>m z{!^INm?J|&7X=;{XveF!(b*=?9NAp4y&r&N3(GKcW4rS(Ejk|Lzs1PrxPI_owB-`H zg3(Rruh^&)`TKA6+_!n>RdI6pw>Vt1_j&+bKIaMTYLiqhZ#y_=J8`TK{Jd<7l9&sY z^^`hmi7^14s16B6)1O;vJWOF$=$B5ONW;;2&|pUvJlmeUS&F;DbSHCrEb0QBDR|my zIs+pE0Y^`qJTyH-_mP=)Y+u^LHcuZhsM3+P||?+W#V!_6E-8boP#R-*na4!o-Q1 zVthtYhK{mDhF(&7Okzo9dTi03X(AE{8cH$JIg%MEQca`S zy@8{Fjft~~BdzWC(di#X{ny;!yYGK9b@=b|zcKZ{vv4D8i+`ilOPl;PJl{!&5-0!w z^fOl#|}vVg%=n)@_e1BrP)`A zKPgs`O0EO}Y2KWLuo`iGaKu1k#YR6BMySxQf2V++Wo{6EHmK>A~Q5o73yM z-RbxC7Qdh0Cz!nG+7BRZE>~FLI-?&W_rJUl-8FDIaXoNBL)@1hwKa^wOr1($*5h~T zF;%f^%<$p8Y_yu(JEg=c_O!aZ#)Gjh$n(hfJAp$C2he555W5zdrBqjFmo|VY+el;o z=*D_w|GXG|p0**hQ7~9-n|y5k%B}TAF0iarDM!q-jYbR^us(>&y;n^2l0C%@2B}KM zyeRT9)oMt97Agvc4sEKUEy%MpXr2vz*lb zh*L}}iG>-pqDRw7ud{=FvTD?}xjD)w{`KzjNom-$jS^;iw0+7nXSnt1R@G|VqoRhE%12nm+PH?9`(4rM0kfrZzIK9JU=^$YNyLvAIoxl#Q)xxDz!^0@zZ zSCs$nfcxK_vRYM34O<1}QHZ|hp4`ioX3x8(UV(FU$J@o%tw3t4k1QPmlEpZa2IujG&(roX_q*%e`Hq|);0;@k z0z=fZiFckp#JzW0p+2A+D$PC~IsakhJJkG(c;CqAgFfU0Z`u$PzG~-9I1oPHrCw&)@s^Dc~^)#HPW0Ra}J^=|h7Fs*<8|b13ZzG6MP*Q1dkoZ6&A^!}|hbjM{2HpqlSXv_UUg1U4gn z3Q)2VjU^ti1myodv+tjhSZp%D978m~p& z43uZUrraHs80Mq&vcetqfQpQP?m!CFj)44t8Z}k`E798wxg&~aCm+DBoI+nKq}&j^ zlPY3W$)K;KtEajks1`G?-@me7C>{PiiBu+41#yU_c(dITaqE?IQ(DBu+c^Ux!>pCj zLC|HJGU*v+!it1(;3e`6igkH(VA)-S+k(*yqxMgUah3$@C zz`7hEM47xr>j8^g`%*f=6S5n>z%Bt_Fg{Tvmr+MIsCx=0gsu_sF`q2hlkEmisz#Fy zj_0;zUWr;Gz}$BS%Y`meb(=$d%@Crs(OoJ|}m#<7=-A~PQbyN$x%2iXP2@e*nO0b7AwfH8cCUa*Wfu@b)D_>I*%uE4O3 z(lfnB`-Xf*LfC)E}e?%X2kK7DItK6Tf<+M^mX0Ijf_!IP>7c8IZX%8_#0060P{QMuV^B9i<^E`_Qf0pv9(P%_s8D`qvDE9LK9u-jB}J2S`(mCO&XHTS04Z5Ez*vl^T%!^$~EH8M-UdwhegL>3IQ*)(MtuH2Xt1p!fS4o~*rR?WLxlA!sjc2(O znjJn~wQ!Fp9s2e^IWP1C<4%sFF}T4omr}7+4asciyo3DntTgWIzhQpQirM$9{EbQd z3jz9vS@{aOqTQHI|l#aUV@2Q^Wko4T0T04Me4!2nsdrA8QY1%fnAYb~d2GDz@lAtfcHq(P7 zaMBAGo}+NcE-K*@9y;Vt3*(aCaMKXBB*BJcD_Qnxpt75r?GeAQ}*|>pYJE=uZb73 zC>sv)18)q#EGrTG6io*}JLuB_jP3AU1Uiu$D7r|2_zlIGb9 zjhst#ni)Y`$)!fc#reM*$~iaYoz~_Cy7J3ZTiPm)E?%`fbk`3Tu-F#`{i!l5pNEn5 zO-Tw-=TojYhzT{J=?SZj=Z8#|eoF>434b-DXiUsignxXNaR3 zm_}4iWU$gt2Mw5NvZ5(VpF`?X*f2UZDs1TEa1oZCif?Jdgr{>O~7}-$|BZ7I(IKW`{f;@|IZFX*R8&iT= zoWstN8&R;}@2Ka%d3vrLtR|O??ben;k8QbS-WB0VgiCz;<$pBmIZdN!aalyCSEm)crpS9dcD^Y@XT1a3+zpi-`D}e#HV<} z$Y(G&o~PvL-xSVD5D?JqF3?B9rxGWeb=oEGJ3vRp5xfBPlngh1O$yI95EL+T8{GC@ z98i1H9KhZGFl|;`)_=QpM6H?eDPpw~^(aFQWwyXZ8_EEE4#@QeT_URray*mEOGsGc z6|sdXtq!hVZo=d#+9^@lm&L5|q&-GDCyUx#YQiccq;spOBe3V+VKdjJA=IL=Zn%P} zNk=_8u}VhzFf{UYZV0`lUwcD&)9AFx0@Fc6LD9A6Rd1=ga>Mi0)_QxM2ddCVRmZ0d z+J=uXc(?5JLX3=)e)Jm$HS2yF`44IKhwRnm2*669_J=2LlwuF5$1tAo@ROSU@-y+;Foy2IEl2^V1N;fk~YR z?&EP8#t&m0B=?aJeuz~lHjAzRBX>&x=A;gIvb>MD{XEV zV%l-+9N-)i;YH%nKP?>f`=?#`>B(`*t`aiPLoQM(a6(qs4p5KFjDBN?8JGrf3z8>= zi7sD)c)Nm~x{e<^jy4nTx${P~cwz_*a>%0_;ULou3kHCAD7EYkw@l$8TN#LO9jC( z1BeFW`k+bu5e8Ns^a8dPcjEVHM;r6UX+cN=Uy7HU)j-myRU0wHd$A1fNI~`4;I~`zC)3ul#8#^rXVSO*m}Ag>c%_;nj=Nv$rCZ z*~L@C@OZg%Q^m)lc-kcX&a*a5`y&DaRxh6O*dfhLfF+fU5wKs(1v*!TkZidw*)YBP za@r`3+^IHRFeO%!ai%rxy;R;;V^Fr=OJlpBX;(b*3+SIw}7= zIq$*Thr(Zft-RlY)D3e8V;BmD&HOfX+E$H#Y@B3?UL5L~_fA-@*IB-!gItK7PIgG9 zgWuGZK_nuZjHVT_Fv(XxtU%)58;W39vzTI2n&)&4Dmq7&JX6G>XFaAR{7_3QB6zsT z?$L8c*WdN~nZGiscY%5KljQARN;`w$gho=p006z;n(qIQ*Zu<``TMO3n0{ARL@gYh zoRwS*|Niw~cR!?hE{m*y@F`1)vx-JRfqET=dJ5_(076st(=lFfjtKHoYg`k3oNmo_ zNbQEw8&sO5jAYmkD|Zaz_yUb0rC})U!rCHOl}JhbYIDLzLvrZVw0~JO`d*6f;X&?V=#T@ND*cv^I;`sFeq4 z##H5;gpZTb^0Hz@3C*~u0AqqNZ-r%rN3KD~%Gw`0XsIq$(^MEb<~H(2*5G^<2(*aI z%7}WB+TRlMIrEK#s0 z93xn*Ohb=kWFc)BNHG4I(~RPn-R8#0lqyBBz5OM6o5|>x9LK@%HaM}}Y5goCQRt2C z{j*2TtT4ne!Z}vh89mjwiSXG=%DURar~=kGNNaO_+Nkb+tRi~Rkf!7a$*QlavziD( z83s4GmQ^Wf*0Bd04f#0HX@ua_d8 z23~z*53ePD6@xwZ(vdl0DLc=>cPIOPOdca&MyR^jhhKrdQO?_jJh`xV3GKz&2lvP8 zEOwW6L*ufvK;TN{=S&R@pzV^U=QNk^Ec}5H z+2~JvEVA{`uMAr)?Kf|aW>33`)UL@bnfIUQc~L;TsTQ6>r-<^rB8uoNOJ>HWgqMI8 zSW}pZmp_;z_2O5_RD|fGyTxaxk53Hg_3Khc<8AUzV|ZeK{fp|Ne933=1&_^Dbv5^u zB9n=*)k*tjHDRJ@$bp9mrh}qFn*s}npMl5BMDC%Hs0M0g-hW~P*3CNG06G!MOPEQ_ zi}Qs-6M8aMt;sL$vlmVBR^+Ry<64jrm1EI1%#j?c?4b*7>)a{aDw#TfTYKq+SjEFA z(aJ&z_0?0JB83D-i3Vh+o|XV4UP+YJ$9Boid2^M2en@APw&wx7vU~t$r2V`F|7Qfo z>WKgI@eNBZ-+Og<{u2ZiG%>YvH2L3fNpV9J;WLJoBZda)01Rn;o@){01{7E#ke(7U zHK>S#qZ(N=aoae*4X!0A{)nu0R_sKpi1{)u>GVjC+b5Jyl6#AoQ-1_3UDovNSo`T> z?c-@7XX*2GMy?k?{g)7?Sv;SJkmxYPJPs!&QqB12ejq`Lee^-cDveVWL^CTUldb(G zjDGe(O4P=S{4fF=#~oAu>LG>wrU^z_?3yt24FOx>}{^lCGh8?vtvY$^hbZ)9I0E3r3NOlb9I?F-Yc=r$*~l`4N^xzlV~N zl~#oc>U)Yjl0BxV>O*Kr@lKT{Z09OXt2GlvE38nfs+DD7exl|&vT;)>VFXJVZp9Np zDK}aO;R3~ag$X*|hRVY3OPax|PG`@_ESc8E!mHRByJbZQRS38V2F__7MW~sgh!a>98Q2%lUNFO=^xU52|?D=IK#QjwBky-C>zOWlsiiM&1n z;!&1((Xn1$9K}xabq~222gYvx3hnZPg}VMF_GV~5ocE=-v>V=T&RsLBo&`)DOyIj* zLV{h)JU_y*7SdRtDajP_Y+rBkNN*1_TXiKwHH2&p51d(#zv~s#HwbNy?<+(=9WBvo zw2hkk2Dj%kTFhY+$T+W-b7@qD!bkfN#Z2ng@Pd=i3-i?xYfs5Z*1hO?kd7Sp^9`;Y zM2jeGg<-nJD1er@Pc_cSY7wo5dzQX44=%6rn}P_SRbpzsA{6B+!$3B0#;}qwO37G^ zL(V_5JK`XT?OHVk|{_$vQ|oNEpab*BO4F zUTNQ7RUhnRsU`TK#~`)$icsvKh~(pl=3p6m98@k3P#~upd=k*u20SNcb{l^1rUa)>qO997)pYRWMncC8A&&MHlbW?7i^7M`+B$hH~Y|J zd>FYOGQ;j>Zc2e7R{KK7)0>>nn_jYJy&o@sK!4G>-rLKM8Hv)f;hi1D2fAc$+six2 zyVZ@wZ6x|fJ!4KrpCJY=!Mq0;)X)OoS~{Lkh6u8J`eK%u0WtKh6B>GW_)PVc zl}-k`p09qwGtZ@VbYJC!>29V?Dr>>vk?)o(x?!z*9DJ||9qG-&G~#kXxbw{KKYy}J zQKa-dPt~M~E}V?PhW0R26xdA%1T*%ra6SguGu50YHngOTIv)@N|YttEXo#OZfgtP7;H?EeZZxo<}3YlYxtBq znJ!WFR^tmGf0Py}N?kZ(#=VtpC@%xJkDmfcCoBTxq zr_|5gP?u1@vJZbxPZ|G0AW4=tpb84gM2DpJU||(b8kMOV1S3|(yuwZJ&rIiFW(U;5 zUtAW`O6F6Zy+eZ1EDuP~AAHlSY-+A_eI5Gx)%*uro5tljy}kCZU*_d7)oJ>oQSZ3* zneTn`{gnNC&uJd)0aMBzAg021?YJ~b(fmkwZAd696a=0NzBAqBN54KuNDwa*no(^O z6p05bioXUR^uXjpTol*ppHp%1v9e)vkoUAUJyBx3lw0UO39b0?^{}yb!$yca(@DUn zCquRF?t=Zb9`Ed3AI6|L{eX~ijVH`VzSMheKoP7LSSf4g>md>`yi!TkoG5P>Ofp+n z(v~rW+(5L96L{vBb^g51B=(o)?%%xhvT*A5btOpw(TKh^g^4c zw>0%X!_0`{iN%RbVk+A^f{w-4-SSf*fu@FhruNL##F~sF24O~u zyYF<3el2b$$wZ_|uW#@Ak+VAGk#e|kS8nL1g>2B-SNMjMp^8;-FfeofY2fphFHO!{ z*!o4oTb{4e;S<|JEs<1_hPsmAlVNk?_5-Fp5KKU&d#FiNW~Y+pVFk@Cua1I{T+1|+ zHx6rFMor)7L)krbilqsWwy@T+g3DiH5MyVf8Wy}XbEaoFIDr~y;@r&I>FMW{ z?Q+(IgyebZ)-i4jNoXQhq4Muy9Fv+OxU;9_Jmn+<`mEC#%2Q_2bpcgzcinygNI!&^ z=V$)o2&Yz04~+&pPWWn`rrWxJ&}8khR)6B(--!9Q zubo}h+1T)>a@c)H^i``@<^j?|r4*{;tQf78(xn0g39IoZw0(CwY1f<%F>kEaJ zp9u|IeMY5mRdAlw*+gSN^5$Q)ShM<~E=(c8QM+T-Qk)FyKz#Sw0EJ*edYcuOtO#~Cx^(M7w5 z3)rl#L)rF|(Vun2LkFr!rg8Q@=r>9p>(t3Gf_auiJ2Xx9HmxYTa|=MH_SUlYL`mz9 zTTS$`%;D-|Jt}AP1&k7PcnfFNTH0A-*FmxstjBDiZX?}%u%Yq94$fUT&z6od+(Uk> zuqsld#G(b$G8tus=M!N#oPd|PVFX)?M?tCD0tS%2IGTfh}3YA3f&UM)W$_GNV8 zQo+a(ml2Km4o6O%gKTCSDNq+#zCTIQ1*`TIJh~k6Gp;htHBFnne))rlFdGqwC6dx2+La1&Mnko*352k0y z+tQcwndQlX`nc6nb$A9?<-o|r*%aWXV#=6PQic0Ok_D;q>wbv&j7cKc!w4~KF#-{6 z(S%6Za)WpGIWf7jZ3svNG5OLs0>vCL9{V7cgO%zevIVMH{WgP*^D9ws&OqA{yr|m| zKD4*07dGXshJHd#e%x%J+qmS^lS|0Bp?{drv;{@{l9ArPO&?Q5=?OO9=}h$oVe#3b z3Yofj&Cb}WC$PxmRRS)H%&$1-)z7jELS}!u!zQ?A^Y{Tv4QVt*vd@uj-^t2fYRzQj zfxGR>-q|o$3sGn^#VzZ!QQx?h9`njeJry}@x?|k0-GTTA4y3t2E`3DZ!A~D?GiJup z)8%PK2^9OVRlP(24P^4_<|D=H^7}WlWu#LgsdHzB%cPy|f8dD3|A^mh4WXxhLTVu_ z@abE{6Saz|Y{rXYPd4$tfPYo}ef(oQWZ=4Bct-=_9`#Qgp4ma$n$`tOwq#&E18$B; z@Bp)bn3&rEi0>fWWZ@7k5WazfoX`SCO4jQWwVuo+$PmSZn^Hz?O(-tW@*DGxuf)V1 zO_xm&;NVCaHD4dqt(-MlszI3F-p?0!-e$fbiCeuaw66h^TTDLWuaV<@C-`=Xe5WL) zwooG7h>4&*)p3pKMS3O!4>-4jQUN}iAMQ)2*70?hP~)TzzR?-f@?Aqy$$1Iy8VGG$ zMM?8;j!pUX7QQD$gRc_#+=raAS577ga-w?jd`vCiN5lu)dEUkkUPl9!?{$IJNxQys z*E4e$eF&n&+AMRQR2gcaFEjAy*r)G!s(P6D&TfoApMFC_*Ftx0|D0@E-=B7tezU@d zZ{hGiN;YLIoSeRS;9o%dEua4b%4R3;$SugDjP$x;Z!M!@QibuSBb)HY!3zJ7M;^jw zlx6AD50FD&p3JyP*>o+t9YWW8(7P2t!VQQ21pHJOcG_SXQD;(5aX#M6x##5H_Re>6lPyDCjxr*R(+HE%c&QN+b^tbT zXBJk?p)zhJj#I?&Y2n&~XiytG9!1ox;bw5Rbj~)7c(MFBb4>IiRATdhg zmiEFlj@S_hwYYI(ki{}&<;_7(Z0Qkfq>am z&LtL=2qc7rWguk3BtE4zL41@#S;NN*-jWw|7Kx7H7~_%7fPt;TIX}Ubo>;Rmj94V> zNB1=;-9AR7s`Pxn}t_6^3ahlq53e&!Lh85uG zec0vJY_6e`tg7LgfrJ3k!DjR)Bi#L@DHIrZ`sK=<5O0Ip!fxGf*OgGSpP@Hbbe&$9 z;ZI}8lEoC2_7;%L2=w?tb%1oL0V+=Z`7b=P&lNGY;yVBazXRYu;+cQDKvm*7NCxu&i;zub zAJh#11%?w>E2rf2e~C4+rAb-&$^vsdACs7 z@|Ra!OfVM(ke{vyiqh7puf&Yp6cd6{DptUteYfIRWG3pI+5< zBVBI_xkBAc<(pcb$!Y%dTW(b;B;2pOI-(QCsLv@U-D1XJ z(Gk8Q3l7Ws46Aktuj>|s{$6zA&xCPuXL-kB`CgYMs}4IeyG*P51IDwW?8UNQd+$i~ zlxOPtSi5L|gJcF@DwmJA5Ju8HEJ>o{{upwIpb!f{2(vLNBw`7xMbvcw<^{Fj@E~1( z?w`iIMieunS#>nXlmUcSMU+D3rX28f?s7z;X=se6bo8;5vM|O^(D6{A9*ChnGH!RG zP##3>LDC3jZPE4PH32AxrqPk|yIIrq~`aL-=}`okhNu9aT%q z1b)7iJ)CN=V#Ly84N_r7U^SH2FGdE5FpTO2 z630TF$P>GNMu8`rOytb(lB2};`;P4YNwW1<5d3Q~AX#P0aX}R2b2)`rgkp#zTxcGj zAV^cvFbhP|JgWrq_e`~exr~sIR$6p5V?o4Wym3kQ3HA+;Pr$bQ0(PmADVO%MKL!^q z?zAM8j1l4jrq|5X+V!8S*2Wl@=7*pPgciTVK6kS1Ge zMsd_u6DFK$jTnvVtE;qa+8(1sGBu~n&F%dh(&c(Zs4Fc#A=gG^^%^AyH}1^?|8quj zl@Z47h$){PlELJgYZCIHHL= z{U8O>Tw4x3<1{?$8>k-P<}1y9DmAZP_;(3Y*{Sk^H^A=_iSJ@+s5ktgwTXz_2$~W9>VVZsfwCm@s0sQ zeB50_yu@uS+e7QoPvdCwDz{prjo(AFwR%C?z`EL{1`|coJHQTk^nX=tvs1<0arUOJ z!^`*x&&BvTYmemyZ)2p~{%eYX=JVR?DYr(rNgqRMA5E1PR1Iw=prk=L2ldy3r3Vg@27IZx43+ywyzr-X*p*d@tZV+!U#~$-q=8c zgdSuh#r?b4GhEGNai)ayHQpk>5(%j5c@C1K3(W1pb~HeHpaqijJZa-e6vq_8t-^M^ zBJxq|MqZc?pjXPIH}70a5vt!IUh;l}<>VX<-Qcv^u@5(@@M2CHSe_hD$VG-eiV^V( zj7*9T0?di?P$FaD6oo?)<)QT>Npf6Og!GO^GmPV(Km0!=+dE&bk#SNI+C9RGQ|{~O*VC+tXK3!n`5 zHfl6>lwf_aEVV3`0T!aHNZLsj$paS$=LL(?b!Czaa5bbSuZ6#$_@LK<(7yrrl+80| z{tOFd=|ta2Z`^ssozD9BINn45NxUeCQis?-BKmU*Kt=FY-NJ+)8S1ecuFtN-M?&42 zl2$G>u!iNhAk*HoJ^4v^9#ORYp5t^wDj6|lx~5w45#E5wVqI1JQ~9l?nPp1YINf++ zMAdSif~_ETv@Er(EFBI^@L4BULFW>)NI+ejHFP*T}UhWNN`I)RRS8za? z*@`1>9ZB}An%aT5K=_2iQmfE;GcBVHLF!$`I99o5GO`O%O_zLr9AG18>&^HkG(;=V z%}c!OBQ~?MX(9h~tajX{=x)+!cbM7$YzTlmsPOdp2L-?GoW`@{lY9U3f;OUo*BwRB z8A+nv(br0-SH#VxGy#ZrgnGD(=@;HME;yd46EgWJ`EL%oXc&lFpc@Y}^>G(W>h_v_ zlN!`idhX+OjL+~T?19sroAFVGfa5tX-D49w$1g2g_-T|EpHL6}K_aX4$K=LTvwtlF zL*z}j{f+Uoe7{-px3_5iKPA<_7W=>Izkk)!l9ez2w%vi(?Y;i8AxRNLSOGDzNoqoI zP!1uAl}r=_871(G?y`i&)-7{u=%nxk7CZ_Qh#!|ITec zwQn`33GTUM`;D2POWnkqngqJhJRlM>CTONzTG}>^Q0wUunQyn|TAiHzyX2_%ATx%P z%7gW)%4rA9^)M<_%k@`Y?RbC<29sWU&5;@|9thf2#zf8z12$hRcZ!CSb>kUp=4N#y zl3hE#y6>kkA8VY2`W`g5Ip?2qC_BY$>R`iGQLhz2-S>x(RuWv)SPaGdl^)gGw7tjR zH@;jwk!jIaCgSg_*9iF|a);sRUTq30(8I(obh^|}S~}P4U^BIGYqcz;MPpC~Y@k_m zaw4WG1_vz2GdCAX!$_a%GHK**@IrHSkGoN>)e}>yzUTm52on`hYot7cB=oA-h1u|R ztH$11t?54Qg2L+i33FPFKKRm1aOjKST{l1*(nps`>sv%VqeVMWjl5+Gh+9);hIP8? zA@$?}Sc z3qIRpba+y5yf{R6G(u8Z^vkg0Fu&D-7?1s=QZU`Ub{-!Y`I?AGf1VNuc^L3v>)>i# z{DV9W$)>34wnzAXUiV^ZpYKw>UElrN_5Xj6{r_3| z$X5PK`e5$7>~9Dj7gK5ash(dvs`vwfk}&RD`>04;j62zoXESkFBklYaKm5seyiX(P zqQ-;XxlV*yg?Dhlx%xt!b0N3GHp@(p$A;8|%# zZ5m2KL|{on4nr>2_s9Yh=r5ScQ0;aMF)G$-9-Ca6%wA`Pa)i?NGFA|#Yi?{X-4ZO_ z^}%7%vkzvUHa$-^Y#aA+aiR5sa%S|Ebyn`EV<3Pc?ax_f>@sBZF1S;7y$CXd5t5=WGsTKBk8$OfH4v|0?0I=Yp}7c=WBSCg!{0n)XmiU;lfx)**zZaYqmDJelxk$)nZyx5`x$6R|fz(;u zEje5Dtm|a%zK!!tk3{i9$I2b{vXNFy%Bf{50X!x{98+BsDr_u9i>G5%*sqEX|06J0 z^IY{UcEbj6LDwuMh7cH`H@9sVt1l1#8kEQ(LyT@&+K}(ReE`ux8gb0r6L_#bDUo^P z3Ka2lRo52Hdtl_%+pwVs14=q`{d^L58PsU@AMf(hENumaxM{7iAT5sYmWh@hQCO^ zK&}ijo=`VqZ#a3vE?`7QW0ZREL17ZvDfdqKGD?0D4fg{7v%|Yj&_jcKJAB)>=*RS* zto8p6@k%;&^ZF>hvXm&$PCuEp{uqw3VPG$9VMdW5$w-fy2CNNT>E;>ejBgy-m_6`& z97L1p{%srn@O_JQgFpa_#f(_)eb#YS>o>q3(*uB;uZb605(iqM$=NK{nHY=+X2*G) zO3-_Xh%aG}fHWe*==58zBwp%&`mge<8uq8;xIxOd=P%9EK!34^E9sk|(Zq1QSz-JVeP12Fp)-`F|KY$LPwUE?rku zY@OJ)Z9A!ojfzfeyJ9;zv2EM7ZQB)AR5xGa-tMn^bl)FmoIiVyJ@!~@%{}qXXD&Ns zPnfe5U+&ohKefILu_1mPfLGuapX@btta5C#gPB2cjk5m4T}Nfi+Vfka!Yd(L?-c~5 z#ZK4VeQEXNPc4r$K00Fg>g#_W!YZ)cJ?JTS<&68_$#cZT-ME`}tcwqg3#``3M3UPvn+pi}(VNNx6y zFIMVb6OwYU(2`at$gHba*qrMVUl8xk5z-z~fb@Q3Y_+aXuEKH}L+>eW__!IAd@V}L zkw#s%H0v2k5-=vh$^vPCuAi22Luu3uKTf6fPo?*nvj$9(u)4$6tvF-%IM+3pt*cgs z_?wW}J7VAA{_~!?))?s6{M=KPpVhg4fNuU*|3THp@_(q!b*hdl{fjRVFWtu^1dV(f z6iOux9hi&+UK=|%M*~|aqFK{Urfl!TA}UWY#`w(0P!KMe1Si{8|o))Gy6d7;!JQYhgMYmXl?3FfOM2nQGN@~Ap6(G z3+d_5y@=nkpKAhRqf{qQ~k7Z$v&l&@m7Ppt#FSNzKPZM z8LhihcE6i=<(#87E|Wr~HKvVWhkll4iSK$^mUHaxgy8*K$_Zj;zJ`L$naPj+^3zTi z-3NTaaKnD5FPY-~?Tq6QHnmDDRxu0mh0D|zD~Y=vv_qig5r-cIbCpxlju&8Sya)@{ zsmv6XUSi)@(?PvItkiZEeN*)AE~I_?#+Ja-r8$(XiXei2d@Hi7Rx8+rZZb?ZLa{;@*EHeRQ-YDadz~M*YCM4&F-r;E#M+@CSJMJ0oU|PQ^ z=E!HBJDMQ2TN*Y(Ag(ynAL8%^v;=~q?s4plA_hig&5Z0x_^Oab!T)@6kRN$)qEJ6E zNuQjg|G7iwU(N8pI@_6==0CL;lRh1dQF#wePhmu@hADFd3B5KIH#dx(2A zp~K&;Xw}F_N6CU~0)QpQk7s$a+LcTOj1%=WXI(U=Dv!6 z{#<#-)2+gCyyv=Jw?Ab#PVkxPDeH|sAxyG`|Ys}A$PW4TdBv%zDz z^?lwrxWR<%Vzc8Sgt|?FL6ej_*e&rhqJZ3Y>k=X(^dytycR;XDU16}Pc9Vn0>_@H+ zQ;a`GSMEG64=JRAOg%~L)x*w{2re6DVprNp+FcNra4VdNjiaF0M^*>CdPkt(m150rCue?FVdL0nFL$V%5y6N z%eLr5%YN7D06k5ji5*p4v$UMM)G??Q%RB27IvH7vYr_^3>1D-M66#MN8tWGw>WED} z5AhlsanO=STFYFs)Il_0i)l)f<8qn|$DW7ZXhf5xI;m+7M5-%P63XFQrG9>DMqHc} zsgNU9nR`b}E^mL5=@7<1_R~j@q_2U^3h|+`7YH-?C=vme1C3m`Fe0HC>pjt6f_XMh zy~-i-8R46QNYneL4t@)<0VU7({aUO?aH`z4V2+kxgH5pYD5)wCh75JqQY)jIPN=U6 z+qi8cGiOtXG2tXm;_CfpH9ESCz#i5B(42}rBJJF$jh<1sbpj^8&L;gzGHb8M{of+} zzF^8VgML2O9nxBW7AvdEt90vp+#kZxWf@A)o9f9}vKJy9NDBjBW zSt=Hcs=YWCwnfY1UYx*+msp{g!w0HC<_SM!VL1(I2PE?CS}r(eh?{I)mQixmo5^p# zV?2R!R@3GV6hwTCrfHiK#3Orj>I!GS2kYhk1S;aFBD_}u2v;0HYFq}Iz1Z(I4oca4 zxquja8$+8JW_EagDHf$a1OTk5S97umGSDaj)gH=fLs9>_=XvVj^Xj9a#gLdk=&3tl zfmK9MNnIX9v{?%xdw7568 zNrZ|roYs(vC4pHB5RJ8>)^*OuyNC>x7ad)tB_}3SgQ96+-JT^Qi<`xi=)_=$Skwv~ zdqeT9Pa`LYvCAn&rMa2aCDV(TMI#PA5g#RtV|CWpgDYRA^|55LLN^uNh*gOU>Z=a06qJ;$C9z8;n-Pq=qZnc1zUwJ@t)L;&NN+E5m zRkQ(SeM8=l-aoAKGKD>!@?mWTW&~)uF2PYUJ;tB^my`r9n|Ly~0c%diYzqs9W#FTjy?h&X3TnH zXqA{QI82sdjPO->f=^K^f>N`+B`q9&rN0bOXO79S&a9XX8zund(kW7O76f4dcWhIu zER`XSMSFbSL>b;Rp#`CuGJ&p$s~G|76){d?xSA5wVg##_O0DrmyEYppyBr%fyWbbv zp`K84JwRNP$d-pJ!Qk|(RMr?*!wi1if-9G#0p>>1QXKXWFy)eB3ai)l3601q8!9JC zvU#ZWWDNKq9g6fYs?JQ)Q4C_cgTy3FhgKb8s&m)DdmL5zhNK#8wWg!J*7G7Qhe9VU zha?^AQTDpYcuN!B+#1dE*X{<#!M%zfUQbj=zLE{dW0XeQ7-oIsGY6RbkP2re@Q{}r_$iiH0xU%iN*ST`A)-EH6eaZB$GA#v)cLi z*MpA(3bYk$oBDKAzu^kJoSUsDd|856DApz={3u8sbQV@JnRkp2nC|)m;#T=DvIL-O zI4vh;g7824l}*`_p@MT4+d`JZ2%6NQh=N9bmgJ#q!hK@_<`HQq3}Z8Ij>3%~<*= zcv=!oT#5xmeGI92lqm9sGVE%#X$ls;St|F#u!?5Y7syhx6q#MVRa&lBmmn%$C0QzU z);*ldgwwCmzM3uglr}!Z2G+?& zf%Dpo&mD%2ZcNFiN-Z0f;c_Q;A%f@>26f?{d1kxIJD}LxsQkB47SAdwinfMILZdN3 zfj^HmTzS3Ku5BxY>ANutS8WPQ-G>v4^_Qndy==P3pDm+Xc?>rUHl-4+^%Sp5atOja z2oP}ftw-rqnb}+khR3CrRg^ibi6?QYk1*i^;kQGirQ=uB9Sd1NTfT-Rbv;hqnY4neE5H1YUrjS2m+2&@uXiAo- zrKUX|Ohg7(6F(AoP~tj;NZlV#xsfo-5reuQHB$&EIAhyZk;bL;k9ouDmJNBAun;H& zn;Of1z_Qj`x&M;5X;{s~iGzBQTY^kv-k{ksbE*Dl%Qf%N@hQCfY~iUw!=F-*$cpf2 z3wix|aLBV0b;W@z^%7S{>9Z^T^fLOI68_;l@+Qzaxo`nAI8emTV@rRhEKZ z?*z_{oGdI~R*#<2{bkz$G~^Qef}$*4OYTgtL$e9q!FY7EqxJ2`zk6SQc}M(k(_MaV zSLJnTXw&@djco1~a(vhBl^&w=$fa9{Sru>7g8SHahv$&Bl(D@(Zwxo_3r=;VH|uc5 zi1Ny)J!<(KN-EcQ(xlw%PNwK8U>4$9nVOhj(y0l9X^vP1TA>r_7WtSExIOsz`nDOP zs}d>Vxb2Vo2e5x8p(n~Y5ggAyvib>d)6?)|E@{FIz?G3PVGLf7-;BxaP;c?7ddH$z zA+{~k^V=bZuXafOv!RPsE1GrR3J2TH9uB=Z67gok+u`V#}BR86hB1xl}H4v`F+mRfr zYhortD%@IGfh!JB(NUNSDh+qDz?4ztEgCz&bIG-Wg7w-ua4ChgQR_c+z8dT3<1?uX z*G(DKy_LTl*Ea!%v!RhpCXW1WJO6F`bgS-SB;Xw9#! z<*K}=#wVu9$`Yo|e!z-CPYH!nj7s9dEPr-E`DXUBu0n!xX~&|%#G=BeM?X@shQQMf zMvr2!y7p_gD5-!Lnm|a@z8Of^EKboZsTMk%5VsJEm>VsJ4W7Kv{<|#4f-qDE$D-W>gWT%z-!qXnDHhOvLk=?^a1*|0j z{pW{M0{#1VcR5;F!!fIlLVNh_Gj zbnW(_j?0c2q$EHIi@fSMR{OUKBcLr{Y&$hrM8XhPByyZaXy|dd&{hYQRJ9@Fn%h3p7*VQolBIV@Eq`=y%5BU~3RPa^$a?ixp^cCg z+}Q*X+CW9~TL29@OOng(#OAOd!)e$d%sr}^KBJ-?-X&|4HTmtemxmp?cT3uA?md4% zT8yZ0U;6Rg6JHy3fJae{6TMGS?ZUX6+gGTT{Q{)SI85$5FD{g-eR%O0KMpWPY`4@O zx!hen1*8^E(*}{m^V_?}(b5k3hYo=T+$&M32+B`}81~KKZhY;2H{7O-M@vbCzuX0n zW-&HXeyr1%I3$@ns-V1~Lb@wIpkmx|8I~ob1Of7i6BTNysEwI}=!nU%q7(V_^+d*G z7G;07m(CRTJup!`cdYi93r^+LY+`M*>aMuHJm(A8_O8C#A*$!Xvddgpjx5)?_EB*q zgE8o5O>e~9IiSC@WtZpF{4Bj2J5eZ>uUzY%TgWF7wdDE!fSQIAWCP)V{;HsU3ap?4 znRsiiDbtN7i9hapO;(|Ew>Ip2TZSvK9Z^N21%J?OiA_&eP1{(Pu_=%JjKy|HOardq ze?zK^K zA%sjF64*Wufad%H<) z^|t>e*h+Z1#l=5wHexzt9HNDNXgM=-OPWKd^5p!~%SIl>Fo&7BvNpbf8{NXmH)o{r zO=aBJ;meX1^{O%q;kqdw*5k!Y7%t_30 zy{nGRVc&5qt?dBwLs+^Sfp;f`YVMSB#C>z^a9@fpZ!xb|b-JEz1LBX7ci)V@W+kvQ89KWA0T~Lj$aCcfW#nD5bt&Y_< z-q{4ZXDqVg?|0o)j1%l0^_it0WF*LCn-+)c!2y5yS7aZIN$>0LqNnkujV*YVes(v$ zY@_-!Q;!ZyJ}Bg|G-~w@or&u0RO?vlt5*9~yeoPV_UWrO2J54b4#{D(D>jF(R88u2 zo#B^@iF_%S>{iXSol8jpmsZuJ?+;epg>k=$d`?GSegAVp3n$`GVDvK${N*#L_1`44 z{w0fL{2%)0|E+qgZtjX}itZz^KJt4Y;*8uSK}Ft38+3>j|K(PxIXXR-t4VopXo#9# zt|F{LWr-?34y`$nLBVV_*UEgA6AUI65dYIbqpNq9cl&uLJ0~L}<=ESlOm?Y-S@L*d z<7vt}`)TW#f%Rp$Q}6@3=j$7Tze@_uZO@aMn<|si{?S}~maII`VTjs&?}jQ4_cut9$)PEqMukwoXobzaKx^MV z2fQwl+;LSZ$qy%Tys0oo^K=jOw$!YwCv^ei4NBVauL)tN%=wz9M{uf{IB(BxK|lT*pFkmNK_1tV`nb%jH=a0~VNq2RCKY(rG7jz!-D^k)Ec)yS%17pE#o6&eY+ z^qN(hQT$}5F(=4lgNQhlxj?nB4N6ntUY6(?+R#B?W3hY_a*)hnr4PA|vJ<6p`K3Z5Hy z{{8(|ux~NLUW=!?9Qe&WXMTAkQnLXg(g=I@(VG3{HE13OaUT|DljyWXPs2FE@?`iU z4GQlM&Q=T<4&v@Fe<+TuXiZQT3G~vZ&^POfmI1K2h6t4eD}Gk5XFGpbj1n_g*{qmD6Xy z`6Vv|lLZtLmrnv*{Q%xxtcWVj3K4M%$bdBk_a&ar{{GWyu#ljM;dII;*jP;QH z#+^o-A4np{@|Mz+LphTD0`FTyxYq#wY)*&Ls5o{0z9yg2K+K7ZN>j1>N&;r+Z`vI| zDzG1LJZ+sE?m?>x{5LJx^)g&pGEpY=fQ-4}{x=ru;}FL$inHemOg%|R*ZXPodU}Kh zFEd5#+8rGq$Y<_?k-}r5zgQ3jRV=ooHiF|@z_#D4pKVEmn5CGV(9VKCyG|sT9nc=U zEoT67R`C->KY8Wp-fEcjjFm^;Cg(ls|*ABVHq8clBE(;~K^b+S>6uj70g? z&{XQ5U&!Z$SO7zfP+y^8XBbiu*Cv-yJG|l-oe*!s5$@Lh_KpxYL2sx`B|V=dETN>5K+C+CU~a_3cI8{vbu$TNVdGf15*>D zz@f{zIlorkY>TRh7mKuAlN9A0>N>SV`X)+bEHms=mfYTMWt_AJtz_h+JMmrgH?mZt zm=lfdF`t^J*XLg7v+iS)XZROygK=CS@CvUaJo&w2W!Wb@aa?~Drtf`JV^cCMjngVZ zv&xaIBEo8EYWuML+vxCpjjY^s1-ahXJzAV6hTw%ZIy!FjI}aJ+{rE&u#>rs)vzuxz z+$5z=7W?zH2>Eb32dvgHYZtCAf!=OLY-pb4>Ae79rd68E2LkVPj-|jFeyqtBCCwiW zkB@kO_(3wFq)7qwV}bA=zD!*@UhT`geq}ITo%@O(Z5Y80nEX~;0-8kO{oB6|(4fQh z);73T!>3@{ZobPwRv*W?7m0Ml9GmJBCJd&6E?hdj9lV= z4flNfsc(J*DyPv?RCOx!MSvk(M952PJ-G|JeVxWVjN~SNS6n-_Ge3Q;TGE;EQvZg86%wZ`MB zSMQua(i*R8a75!6$QRO^(o7sGoomb+Y{OMy;m~Oa`;P9Yqo>?bJAhqXxLr7_3g_n>f#UVtxG!^F#1+y@os6x(sg z^28bsQ@8rw%Gxk-stAEPRbv^}5sLe=VMbkc@Jjimqjvmd!3E7+QnL>|(^3!R} zD-l1l7*Amu@j+PWLGHXXaFG0Ct2Q=}5YNUxEQHCAU7gA$sSC<5OGylNnQUa>>l%sM zyu}z6i&({U@x^hln**o6r2s-(C-L50tQvz|zHTqW!ir?w&V23tuYEDJVV#5pE|OJu z7^R!A$iM$YCe?8n67l*J-okwfZ+ZTkGvZ)tVPfR;|3gyFjF)8V zyXXN=!*bpyRg9#~Bg1+UDYCt0 ztp4&?t1X0q>uz;ann$OrZs{5*r`(oNvw=$7O#rD|Wuv*wIi)4b zGtq4%BX+kkagv3F9Id6~-c+1&?zny%w5j&nk9SQfo0k4LhdSU_kWGW7axkfpgR`8* z!?UTG*Zi_baA1^0eda8S|@&F z{)Rad0kiLjB|=}XFJhD(S3ssKlveFFmkN{Vl^_nb!o5M!RC=m)V&v2%e?ZoRC@h3> zJ(?pvToFd`*Zc@HFPL#=otWKwtuuQ_dT-Hr{S%pQX<6dqVJ8;f(o)4~VM_kEQkMR+ zs1SCVi~k>M`u1u2xc}>#D!V&6nOOh-E$O&SzYrjJdZpaDv1!R-QGA141WjQe2s0J~ zQ;AXG)F+K#K8_5HVqRoRM%^EduqOnS(j2)|ctA6Q^=|s_WJYU;Z%5bHp08HPL`YF2 zR)Ad1z{zh`=sDs^&V}J z%$Z$!jd7BY5AkT?j`eqMs%!Gm@T8)4w3GYEX~IwgE~`d|@T{WYHkudy(47brgHXx& zBL1yFG6!!!VOSmDxBpefy2{L_u5yTwja&HA!mYA#wg#bc-m%~8aRR|~AvMnind@zs zy>wkShe5&*un^zvSOdlVu%kHsEo>@puMQ`b1}(|)l~E{5)f7gC=E$fP(FC2=F<^|A zxeIm?{EE!3sO!Gr7e{w)Dx(uU#3WrFZ>ibmKSQ1tY?*-Nh1TDHLe+k*;{Rp!Bmd_m zb#^kh`Y*8l|9Cz2e{;RL%_lg{#^Ar+NH|3z*Zye>!alpt{z;4dFAw^^H!6ING*EFc z_yqhr8d!;%nHX9AKhFQZBGrSzfzYCi%C!(Q5*~hX>)0N`vbhZ@N|i;_972WSx*>LH z87?en(;2_`{_JHF`Sv6Wlps;dCcj+8IJ8ca6`DsOQCMb3n# z3)_w%FuJ3>fjeOOtWyq)ag|PmgQbC-s}KRHG~enBcIwqIiGW8R8jFeBNY9|YswRY5 zjGUxdGgUD26wOpwM#8a!Nuqg68*dG@VM~SbOroL_On0N6QdT9?)NeB3@0FCC?Z|E0 z6TPZj(AsPtwCw>*{eDEE}Gby>0q{*lI+g2e&(YQrsY&uGM{O~}(oM@YWmb*F zA0^rr5~UD^qmNljq$F#ARXRZ1igP`MQx4aS6*MS;Ot(1L5jF2NJ;de!NujUYg$dr# z=TEL_zTj2@>ZZN(NYCeVX2==~=aT)R30gETO{G&GM4XN<+!&W&(WcDP%oL8PyIVUC zs5AvMgh6qr-2?^unB@mXK*Dbil^y-GTC+>&N5HkzXtozVf93m~xOUHn8`HpX=$_v2 z61H;Z1qK9o;>->tb8y%#4H)765W4E>TQ1o0PFj)uTOPEvv&}%(_mG0ISmyhnQV33Z$#&yd{ zc{>8V8XK$3u8}04CmAQ#I@XvtmB*s4t8va?-IY4@CN>;)mLb_4!&P3XSw4pA_NzDb zORn!blT-aHk1%Jpi>T~oGLuh{DB)JIGZ9KOsciWs2N7mM1JWM+lna4vkDL?Q)z_Ct z`!mi0jtr+4*L&N7jk&LodVO#6?_qRGVaucqVB8*us6i3BTa^^EI0x%EREQSXV@f!lak6Wf1cNZ8>*artIJ(ADO*=<-an`3zB4d*oO*8D1K!f z*A@P1bZCNtU=p!742MrAj%&5v%Xp_dSX@4YCw%F|%Dk=u|1BOmo)HsVz)nD5USa zR~??e61sO(;PR)iaxK{M%QM_rIua9C^4ppVS$qCT9j2%?*em?`4Z;4@>I(c%M&#cH z>4}*;ej<4cKkbCAjjDsyKS8rIm90O)Jjgyxj5^venBx&7B!xLmzxW3jhj7sR(^3Fz z84EY|p1NauwXUr;FfZjdaAfh%ivyp+^!jBjJuAaKa!yCq=?T_)R!>16?{~p)FQ3LDoMyG%hL#pR!f@P%*;#90rs_y z@9}@r1BmM-SJ#DeuqCQk=J?ixDSwL*wh|G#us;dd{H}3*-Y7Tv5m=bQJMcH+_S`zVtf;!0kt*(zwJ zs+kedTm!A}cMiM!qv(c$o5K%}Yd0|nOd0iLjus&;s0Acvoi-PFrWm?+q9f^FslxGi z6ywB`QpL$rJzWDg(4)C4+!2cLE}UPCTBLa*_=c#*$b2PWrRN46$y~yST3a2$7hEH= zNjux+wna^AzQ=KEa_5#9Ph=G1{S0#hh1L3hQ`@HrVnCx{!fw_a0N5xV(iPdKZ-HOM za)LdgK}1ww*C_>V7hbQnTzjURJL`S%`6nTHcgS+dB6b_;PY1FsrdE8(2K6FN>37!62j_cBlui{jO^$dPkGHV>pXvW0EiOA zqW`YaSUBWg_v^Y5tPJfWLcLpsA8T zG)!x>pKMpt!lv3&KV!-um= zKCir6`bEL_LCFx4Z5bAFXW$g3Cq`?Q%)3q0r852XI*Der*JNuKUZ`C{cCuu8R8nkt z%pnF>R$uY8L+D!V{s^9>IC+bmt<05h**>49R*#vpM*4i0qRB2uPbg8{{s#9yC;Z18 zD7|4m<9qneQ84uX|J&f-g8a|nFKFt34@Bt{CU`v(SYbbn95Q67*)_Esl_;v291s=9 z+#2F2apZU4Tq=x+?V}CjwD(P=U~d<=mfEFuyPB`Ey82V9G#Sk8H_Ob_RnP3s?)S_3 zr%}Pb?;lt_)Nf>@zX~D~TBr;-LS<1I##8z`;0ZCvI_QbXNh8Iv)$LS=*gHr;}dgb=w5$3k2la1keIm|=7<-JD>)U%=Avl0Vj@+&vxn zt-)`vJxJr88D&!}2^{GPXc^nmRf#}nb$4MMkBA21GzB`-Or`-3lq^O^svO7Vs~FdM zv`NvzyG+0T!P8l_&8gH|pzE{N(gv_tgDU7SWeiI-iHC#0Ai%Ixn4&nt{5y3(GQs)i z&uA;~_0shP$0Wh0VooIeyC|lak__#KVJfxa7*mYmZ22@(<^W}FdKjd*U1CqSjNKW% z*z$5$=t^+;Ui=MoDW~A7;)Mj%ibX1_p4gu>RC}Z_pl`U*{_z@+HN?AF{_W z?M_X@o%w8fgFIJ$fIzBeK=v#*`mtY$HC3tqw7q^GCT!P$I%=2N4FY7j9nG8aIm$c9 zeKTxVKN!UJ{#W)zxW|Q^K!3s;(*7Gbn;e@pQBCDS(I|Y0euK#dSQ_W^)sv5pa%<^o zyu}3d?Lx`)3-n5Sy9r#`I{+t6x%I%G(iewGbvor&I^{lhu-!#}*Q3^itvY(^UWXgvthH52zLy&T+B)Pw;5>4D6>74 zO_EBS)>l!zLTVkX@NDqyN2cXTwsUVao7$HcqV2%t$YzdAC&T)dwzExa3*kt9d(}al zA~M}=%2NVNUjZiO7c>04YH)sRelXJYpWSn^aC$|Ji|E13a^-v2MB!Nc*b+=KY7MCm zqIteKfNkONq}uM;PB?vvgQvfKLPMB8u5+Am=d#>g+o&Ysb>dX9EC8q?D$pJH!MTAqa=DS5$cb+;hEvjwVfF{4;M{5U&^_+r zvZdu_rildI!*|*A$TzJ&apQWV@p{!W`=?t(o0{?9y&vM)V)ycGSlI3`;ps(vf2PUq zX745#`cmT*ra7XECC0gKkpu2eyhFEUb?;4@X7weEnLjXj_F~?OzL1U1L0|s6M+kIhmi%`n5vvDALMagi4`wMc=JV{XiO+^ z?s9i7;GgrRW{Mx)d7rj)?(;|b-`iBNPqdwtt%32se@?w4<^KU&585_kZ=`Wy^oLu9 z?DQAh5z%q;UkP48jgMFHTf#mj?#z|=w= z(q6~17Vn}P)J3M?O)x))%a5+>TFW3No~TgP;f}K$#icBh;rSS+R|}l鯊%1Et zwk~hMkhq;MOw^Q5`7oC{CUUyTw9x>^%*FHx^qJw(LB+E0WBX@{Ghw;)6aA-KyYg8p z7XDveQOpEr;B4je@2~usI5BlFadedX^ma{b{ypd|RNYqo#~d*mj&y`^iojR}s%~vF z(H!u`yx68D1Tj(3(m;Q+Ma}s2n#;O~bcB1`lYk%Irx60&-nWIUBr2x&@}@76+*zJ5 ze&4?q8?m%L9c6h=J$WBzbiTf1Z-0Eb5$IZs>lvm$>1n_Mezp*qw_pr8<8$6f)5f<@ zyV#tzMCs51nTv_5ca`x`yfE5YA^*%O_H?;tWYdM_kHPubA%vy47i=9>Bq) zRQ&0UwLQHeswmB1yP)+BiR;S+Vc-5TX84KUA;8VY9}yEj0eESSO`7HQ4lO z4(CyA8y1G7_C;6kd4U3K-aNOK!sHE}KL_-^EDl(vB42P$2Km7$WGqNy=%fqB+ zSLdrlcbEH=T@W8V4(TgoXZ*G1_aq$K^@ek=TVhoKRjw;HyI&coln|uRr5mMOy2GXP zwr*F^Y|!Sjr2YQXX(Fp^*`Wk905K%$bd03R4(igl0&7IIm*#f`A!DCarW9$h$z`kYk9MjjqN&5-DsH@8xh63!fTNPxWsFQhNv z#|3RjnP$Thdb#Ys7M+v|>AHm0BVTw)EH}>x@_f4zca&3tXJhTZ8pO}aN?(dHo)44Z z_5j+YP=jMlFqwvf3lq!57-SAuRV2_gJ*wsR_!Y4Z(trO}0wmB9%f#jNDHPdQGHFR; zZXzS-$`;7DQ5vF~oSgP3bNV$6Z(rwo6W(U07b1n3UHqml>{=6&-4PALATsH@Bh^W? z)ob%oAPaiw{?9HfMzpGb)@Kys^J$CN{uf*HX?)z=g`J(uK1YO^8~s1(ZIbG%Et(|q z$D@_QqltVZu9Py4R0Ld8!U|#`5~^M=b>fnHthzKBRr=i+w@0Vr^l|W;=zFT#PJ?*a zbC}G#It}rQP^Ait^W&aa6B;+0gNvz4cWUMzpv(1gvfw-X4xJ2Sv;mt;zb2Tsn|kSS zo*U9N?I{=-;a-OybL4r;PolCfiaL=y@o9{%`>+&FI#D^uy#>)R@b^1ue&AKKwuI*` zx%+6r48EIX6nF4o;>)zhV_8(IEX})NGU6Vs(yslrx{5fII}o3SMHW7wGtK9oIO4OM&@@ECtXSICLcPXoS|{;=_yj>hh*%hP27yZwOmj4&Lh z*Nd@OMkd!aKReoqNOkp5cW*lC)&C$P?+H3*%8)6HcpBg&IhGP^77XPZpc%WKYLX$T zsSQ$|ntaVVOoRat$6lvZO(G-QM5s#N4j*|N_;8cc2v_k4n6zx9c1L4JL*83F-C1Cn zaJhd;>rHXB%%ZN=3_o3&Qd2YOxrK~&?1=UuN9QhL$~OY-Qyg&})#ez*8NpQW_*a&kD&ANjedxT0Ar z<6r{eaVz3`d~+N~vkMaV8{F?RBVemN(jD@S8qO~L{rUw#=2a$V(7rLE+kGUZ<%pdr z?$DP|Vg#gZ9S}w((O2NbxzQ^zTot=89!0^~hE{|c9q1hVzv0?YC5s42Yx($;hAp*E zyoGuRyphQY{Q2ee0Xx`1&lv(l-SeC$NEyS~8iil3_aNlnqF_G|;zt#F%1;J)jnPT& z@iU0S;wHJ2$f!juqEzPZeZkjcQ+Pa@eERSLKsWf=`{R@yv7AuRh&ALRTAy z8=g&nxsSJCe!QLchJ=}6|LshnXIK)SNd zRkJNiqHwKK{SO;N5m5wdL&qK`v|d?5<4!(FAsDxR>Ky#0#t$8XCMptvNo?|SY?d8b z`*8dVBlXTUanlh6n)!EHf2&PDG8sXNAt6~u-_1EjPI1|<=33T8 zEnA00E!`4Ave0d&VVh0e>)Dc}=FfAFxpsC1u9ATfQ`-Cu;mhc8Z>2;uyXtqpLb7(P zd2F9<3cXS} znMg?{&8_YFTGRQZEPU-XPq55%51}RJpw@LO_|)CFAt62-_!u_Uq$csc+7|3+TV_!h z+2a7Yh^5AA{q^m|=KSJL+w-EWDBc&I_I1vOr^}P8i?cKMhGy$CP0XKrQzCheG$}G# zuglf8*PAFO8%xop7KSwI8||liTaQ9NCAFarr~psQt)g*pC@9bORZ>m`_GA`_K@~&% zijH0z;T$fd;-Liw8%EKZas>BH8nYTqsK7F;>>@YsE=Rqo?_8}UO-S#|6~CAW0Oz1} z3F(1=+#wrBJh4H)9jTQ_$~@#9|Bc1Pd3rAIA_&vOpvvbgDJOM(yNPhJJq2%PCcMaI zrbe~toYzvkZYQ{ea(Wiyu#4WB#RRN%bMe=SOk!CbJZv^m?Flo5p{W8|0i3`hI3Np# zvCZqY%o258CI=SGb+A3yJe~JH^i{uU`#U#fvSC~rWTq+K`E%J@ zasU07&pB6A4w3b?d?q}2=0rA#SA7D`X+zg@&zm^iA*HVi z009#PUH<%lk4z~p^l0S{lCJk1Uxi=F4e_DwlfHA`X`rv(|JqWKAA5nH+u4Da+E_p+ zVmH@lg^n4ixs~*@gm_dgQ&eDmE1mnw5wBz9Yg?QdZwF|an67Xd*x!He)Gc8&2!urh z4_uXzbYz-aX)X1>&iUjGp;P1u8&7TID0bTH-jCL&Xk8b&;;6p2op_=y^m@Nq*0{#o!!A;wNAFG@0%Z9rHo zcJs?Th>Ny6+hI`+1XoU*ED$Yf@9f91m9Y=#N(HJP^Y@ZEYR6I?oM{>&Wq4|v0IB(p zqX#Z<_3X(&{H+{3Tr|sFy}~=bv+l=P;|sBz$wk-n^R`G3p0(p>p=5ahpaD7>r|>pm zv;V`_IR@tvZreIuv2EM7ZQHhO+qUgw#kOs%*ekY^n|=1#x9&c;Ro&I~{rG-#_3ZB1 z?|9}IFdbP}^DneP*T-JaoYHt~r@EfvnPE5EKUwIxjPbsr$% zfWW83pgWST7*B(o=kmo)74$8UU)v0{@4DI+ci&%=#90}!CZz|rnH+Mz=HN~97G3~@ z;v5(9_2%eca(9iu@J@aqaMS6*$TMw!S>H(b z4(*B!|H|8&EuB%mITr~O?vVEf%(Gr)6E=>H~1VR z&1YOXluJSG1!?TnT)_*YmJ*o_Q@om~(GdrhI{$Fsx_zrkupc#y{DK1WOUR>tk>ZE) ziOLoBkhZZ?0Uf}cm>GsA>Rd6V8@JF)J*EQlQ<=JD@m<)hyElXR0`pTku*3MU`HJn| zIf7$)RlK^pW-$87U;431;Ye4Ie+l~_B3*bH1>*yKzn23cH0u(i5pXV! z4K?{3oF7ZavmmtTq((wtml)m6i)8X6ot_mrE-QJCW}Yn!(3~aUHYG=^fA<^~`e3yc z-NWTb{gR;DOUcK#zPbN^D*e=2eR^_!(!RKkiwMW@@yYtEoOp4XjOGgzi`;=8 zi3`Ccw1%L*y(FDj=C7Ro-V?q)-%p?Ob2ZElu`eZ99n14-ZkEV#y5C+{Pq87Gu3&>g zFy~Wk7^6v*)4pF3@F@rE__k3ikx(hzN3@e*^0=KNA6|jC^B5nf(XaoQaZN?Xi}Rn3 z$8&m*KmWvPaUQ(V<#J+S&zO|8P-#!f%7G+n_%sXp9=J%Z4&9OkWXeuZN}ssgQ#Tcj z8p6ErJQJWZ+fXLCco=RN8D{W%+*kko*2-LEb))xcHwNl~Xmir>kmAxW?eW50Osw3# zki8Fl$#fvw*7rqd?%E?}ZX4`c5-R&w!Y0#EBbelVXSng+kUfeUiqofPehl}$ormli zg%r)}?%=?_pHb9`Cq9Z|B`L8b>(!+8HSX?`5+5mm81AFXfnAt1*R3F z%b2RPIacKAddx%JfQ8l{3U|vK@W7KB$CdLqn@wP^?azRks@x8z59#$Q*7q!KilY-P zHUbs(IFYRGG1{~@RF;Lqyho$~7^hNC`NL3kn^Td%A7dRgr_&`2k=t+}D-o9&C!y^? z6MsQ=tc3g0xkK(O%DzR9nbNB(r@L;1zQrs8mzx&4dz}?3KNYozOW5;=w18U6$G4U2 z#2^qRLT*Mo4bV1Oeo1PKQ2WQS2Y-hv&S|C7`xh6=Pj7MNLC5K-zokZ67S)C;(F0Dd zloDK2_o1$Fmza>EMj3X9je7e%Q`$39Dk~GoOj89-6q9|_WJlSl!!+*{R=tGp z8u|MuSwm^t7K^nUe+^0G3dkGZr3@(X+TL5eah)K^Tn zXEtHmR9UIaEYgD5Nhh(s*fcG_lh-mfy5iUF3xxpRZ0q3nZ=1qAtUa?(LnT9I&~uxX z`pV?+=|-Gl(kz?w!zIieXT}o}7@`QO>;u$Z!QB${a08_bW0_o@&9cjJUXzVyNGCm8 zm=W+$H!;_Kzp6WQqxUI;JlPY&`V}9C$8HZ^m?NvI*JT@~BM=()T()Ii#+*$y@lTZBkmMMda>7s#O(1YZR+zTG@&}!EXFG{ zEWPSDI5bFi;NT>Yj*FjH((=oe%t%xYmE~AGaOc4#9K_XsVpl<4SP@E!TgC0qpe1oi zNpxU2b0(lEMcoibQ-G^cxO?ySVW26HoBNa;n0}CWL*{k)oBu1>F18X061$SP{Gu67 z-v-Fa=Fl^u3lnGY^o5v)Bux}bNZ~ z5pL+7F_Esoun8^5>z8NFoIdb$sNS&xT8_|`GTe8zSXQzs4r^g0kZjg(b0bJvz`g<70u9Z3fQILX1Lj@;@+##bP|FAOl)U^9U>0rx zGi)M1(Hce)LAvQO-pW!MN$;#ZMX?VE(22lTlJrk#pB0FJNqVwC+*%${Gt#r_tH9I_ z;+#)#8cWAl?d@R+O+}@1A^hAR1s3UcW{G+>;X4utD2d9X(jF555}!TVN-hByV6t+A zdFR^aE@GNNgSxxixS2p=on4(+*+f<8xrwAObC)D5)4!z7)}mTpb7&ofF3u&9&wPS< zB62WHLGMhmrmOAgmJ+|c>qEWTD#jd~lHNgT0?t-p{T=~#EMcB| z=AoDKOL+qXCfk~F)-Rv**V}}gWFl>liXOl7Uec_8v)(S#av99PX1sQIVZ9eNLkhq$ zt|qu0b?GW_uo}TbU8!jYn8iJeIP)r@;!Ze_7mj{AUV$GEz6bDSDO=D!&C9!M@*S2! zfGyA|EPlXGMjkH6x7OMF?gKL7{GvGfED=Jte^p=91FpCu)#{whAMw`vSLa`K#atdN zThnL+7!ZNmP{rc=Z>%$meH;Qi1=m1E3Lq2D_O1-X5C;!I0L>zur@tPAC9*7Jeh)`;eec}1`nkRP(%iv-`N zZ@ip-g|7l6Hz%j%gcAM}6-nrC8oA$BkOTz^?dakvX?`^=ZkYh%vUE z9+&)K1UTK=ahYiaNn&G5nHUY5niLGus@p5E2@RwZufRvF{@$hW{;{3QhjvEHMvduO z#Wf-@oYU4ht?#uP{N3utVzV49mEc9>*TV_W2TVC`6+oI)zAjy$KJrr=*q##&kobiQ z1vNbya&OVjK`2pdRrM?LuK6BgrLN7H_3m z!qpNKg~87XgCwb#I=Q&0rI*l$wM!qTkXrx1ko5q-f;=R2fImRMwt5Qs{P*p^z@9ex z`2#v(qE&F%MXlHpdO#QEZyZftn4f05ab^f2vjxuFaat2}jke{j?5GrF=WYBR?gS(^ z9SBiNi}anzBDBRc+QqizTTQuJrzm^bNA~A{j%ugXP7McZqJ}65l10({wk++$=e8O{ zxWjG!Qp#5OmI#XRQQM?n6?1ztl6^D40hDJr?4$Wc&O_{*OfMfxe)V0=e{|N?J#fgE>j9jAajze$iN!*yeF%jJU#G1c@@rm zolGW!j?W6Q8pP=lkctNFdfgUMg92wlM4E$aks1??M$~WQfzzzXtS)wKrr2sJeCN4X zY(X^H_c^PzfcO8Bq(Q*p4c_v@F$Y8cHLrH$`pJ2}=#*8%JYdqsqnGqEdBQMpl!Ot04tUGSXTQdsX&GDtjbWD=prcCT9(+ z&UM%lW%Q3yrl1yiYs;LxzIy>2G}EPY6|sBhL&X&RAQrSAV4Tlh2nITR?{6xO9ujGu zr*)^E`>o!c=gT*_@6S&>0POxcXYNQd&HMw6<|#{eSute2C3{&h?Ah|cw56-AP^f8l zT^kvZY$YiH8j)sk7_=;gx)vx-PW`hbSBXJGCTkpt;ap(}G2GY=2bbjABU5)ty%G#x zAi07{Bjhv}>OD#5zh#$0w;-vvC@^}F! z#X$@)zIs1L^E;2xDAwEjaXhTBw2<{&JkF*`;c3<1U@A4MaLPe{M5DGGkL}#{cHL%* zYMG+-Fm0#qzPL#V)TvQVI|?_M>=zVJr9>(6ib*#z8q@mYKXDP`k&A4A};xMK0h=yrMp~JW{L?mE~ph&1Y1a#4%SO)@{ zK2juwynUOC)U*hVlJU17%llUxAJFuKZh3K0gU`aP)pc~bE~mM!i1mi!~LTf>1Wp< zuG+ahp^gH8g8-M$u{HUWh0m^9Rg@cQ{&DAO{PTMudV6c?ka7+AO& z746QylZ&Oj`1aqfu?l&zGtJnpEQOt;OAFq19MXTcI~`ZcoZmyMrIKDFRIDi`FH)w; z8+*8tdevMDv*VtQi|e}CnB_JWs>fhLOH-+Os2Lh!&)Oh2utl{*AwR)QVLS49iTp{6 z;|172Jl!Ml17unF+pd+Ff@jIE-{Oxv)5|pOm@CkHW?{l}b@1>Pe!l}VccX#xp@xgJ zyE<&ep$=*vT=}7vtvif0B?9xw_3Gej7mN*dOHdQPtW5kA5_zGD zpA4tV2*0E^OUimSsV#?Tg#oiQ>%4D@1F5@AHwT8Kgen$bSMHD3sXCkq8^(uo7CWk`mT zuslYq`6Yz;L%wJh$3l1%SZv#QnG3=NZ=BK4yzk#HAPbqXa92;3K5?0kn4TQ`%E%X} z&>Lbt!!QclYKd6+J7Nl@xv!uD%)*bY-;p`y^ZCC<%LEHUi$l5biu!sT3TGGSTPA21 zT8@B&a0lJHVn1I$I3I1I{W9fJAYc+8 zVj8>HvD}&O`TqU2AAb={?eT;0hyL(R{|h23=4fDSZKC32;wWxsVj`P z3J3{M$PwdH!ro*Cn!D&=jnFR>BNGR<<|I8CI@+@658Dy(lhqbhXfPTVecY@L8%`3Q z1Fux2w?2C3th60jI~%OC9BtpNF$QPqcG+Pz96qZJ71_`0o0w_q7|h&O>`6U+^BA&5 zXd5Zp1Xkw~>M%RixTm&OqpNl8Q+ue=92Op_>T~_9UON?ZM2c0aGm=^A4ejrXj3dV9 zhh_bCt-b9`uOX#cFLj!vhZ#lS8Tc47OH>*)y#{O9?AT~KR9LntM|#l#Dlm^8{nZdk zjMl#>ZM%#^nK2TPzLcKxqx24P7R1FPlBy7LSBrRvx>fE$9AJ;7{PQm~^LBX^k#6Zq zw*Z(zJC|`!6_)EFR}8|n8&&Rbj8y028~P~sFXBFRt+tmqH-S3<%N;C&WGH!f3{7cm zy_fCAb9@HqaXa1Y5vFbxWf%#zg6SI$C+Uz5=CTO}e|2fjWkZ;Dx|84Ow~bkI=LW+U zuq;KSv9VMboRvs9)}2PAO|b(JCEC_A0wq{uEj|3x@}*=bOd zwr{TgeCGG>HT<@Zeq8y}vTpwDg#UBvD)BEs@1KP$^3$sh&_joQPn{hjBXmLPJ{tC) z*HS`*2+VtJO{|e$mM^|qv1R*8i(m1`%)}g=SU#T#0KlTM2RSvYUc1fP+va|4;5}Bfz98UvDCpq7}+SMV&;nX zQw~N6qOX{P55{#LQkrZk(e5YGzr|(B;Q;ju;2a`q+S9bsEH@i1{_Y0;hWYn1-79jl z5c&bytD*k)GqrVcHn6t-7kinadiD>B{Tl`ZY@`g|b~pvHh5!gKP4({rp?D0aFd_cN zhHRo4dd5^S6ViN(>(28qZT6E>??aRhc($kP`>@<+lIKS5HdhjVU;>f7<4))E*5|g{ z&d1}D|vpuV^eRj5j|xx9nwaCxXFG?Qbjn~_WSy=N}P0W>MP zG-F%70lX5Xr$a)2i6?i|iMyM|;Jtf*hO?=Jxj12oz&>P=1#h~lf%#fc73M2_(SUM- zf&qnjS80|_Y0lDgl&I?*eMumUklLe_=Td!9G@eR*tcPOgIShJipp3{A10u(4eT~DY zHezEj8V+7m!knn7)W!-5QI3=IvC^as5+TW1@Ern@yX| z7Nn~xVx&fGSr+L%4iohtS3w^{-H1A_5=r&x8}R!YZvp<2T^YFvj8G_vm}5q;^UOJf ztl=X3iL;;^^a#`t{Ae-%5Oq{?M#s6Npj+L(n-*LMI-yMR{)qki!~{5z{&`-iL}lgW zxo+tnvICK=lImjV$Z|O_cYj_PlEYCzu-XBz&XC-JVxUh9;6*z4fuBG+H{voCC;`~GYV|hj%j_&I zDZCj>Q_0RCwFauYoVMiUSB+*Mx`tg)bWmM^SwMA+?lBg12QUF_x2b)b?qb88K-YUd z0dO}3k#QirBV<5%jL$#wlf!60dizu;tsp(7XLdI=eQs?P`tOZYMjVq&jE)qK*6B^$ zBe>VvH5TO>s>izhwJJ$<`a8fakTL!yM^Zfr2hV9`f}}VVUXK39p@G|xYRz{fTI+Yq z20d=)iwjuG9RB$%$^&8#(c0_j0t_C~^|n+c`Apu|x7~;#cS-s=X1|C*YxX3ailhg_|0`g!E&GZJEr?bh#Tpb8siR=JxWKc{#w7g zWznLwi;zLFmM1g8V5-P#RsM@iX>TK$xsWuujcsVR^7TQ@!+vCD<>Bk9tdCo7Mzgq5 zv8d>dK9x8C@Qoh01u@3h0X_`SZluTb@5o;{4{{eF!-4405x8X7hewZWpz z2qEi4UTiXTvsa(0X7kQH{3VMF>W|6;6iTrrYD2fMggFA&-CBEfSqPlQDxqsa>{e2M z(R5PJ7uOooFc|9GU0ELA%m4&4Ja#cQpNw8i8ACAoK6?-px+oBl_yKmenZut#Xumjz zk8p^OV2KY&?5MUwGrBOo?ki`Sxo#?-Q4gw*Sh0k`@ zFTaYK2;}%Zk-68`#5DXU$2#=%YL#S&MTN8bF+!J2VT6x^XBci6O)Q#JfW{YMz) zOBM>t2rSj)n#0a3cjvu}r|k3od6W(SN}V-cL?bi*Iz-8uOcCcsX0L>ZXjLqk zZu2uHq5B|Kt>e+=pPKu=1P@1r9WLgYFq_TNV1p9pu0erHGd!+bBp!qGi+~4A(RsYN@CyXNrC&hxGmW)u5m35OmWwX`I+0yByglO`}HC4nGE^_HUs^&A(uaM zKPj^=qI{&ayOq#z=p&pnx@@k&I1JI>cttJcu@Ihljt?6p^6{|ds`0MoQwp+I{3l6` zB<9S((RpLG^>=Kic`1LnhpW2=Gu!x`m~=y;A`Qk!-w`IN;S8S930#vBVMv2vCKi}u z6<-VPrU0AnE&vzwV(CFC0gnZYcpa-l5T0ZS$P6(?9AM;`Aj~XDvt;Jua=jIgF=Fm? zdp=M$>`phx%+Gu};;-&7T|B1AcC#L4@mW5SV_^1BRbo6;2PWe$r+npRV`yc;T1mo& z+~_?7rA+(Um&o@Tddl zL_hxvWk~a)yY}%j`Y+200D%9$bWHy&;(yj{jpi?Rtz{J66ANw)UyPOm;t6FzY3$hx zcn)Ir79nhFvNa7^a{SHN7XH*|Vlsx`CddPnA&Qvh8aNhEA;mPVv;Ah=k<*u!Zq^7 z<=xs*iQTQOMMcg|(NA_auh@x`3#_LFt=)}%SQppP{E>mu_LgquAWvh<>L7tf9+~rO znwUDS52u)OtY<~!d$;m9+87aO+&`#2ICl@Y>&F{jI=H(K+@3M1$rr=*H^dye#~TyD z!){#Pyfn+|ugUu}G;a~!&&0aqQ59U@UT3|_JuBlYUpT$2+11;}JBJ`{+lQN9T@QFY z5+`t;6(TS0F?OlBTE!@7D`8#URDNqx2t6`GZ{ZgXeS@v%-eJzZOHz18aS|svxII$a zZeFjrJ*$IwX$f-Rzr_G>xbu@euGl)B7pC&S+CmDJBg$BoV~jxSO#>y z33`bupN#LDoW0feZe0%q8un0rYN|eRAnwDHQ6e_)xBTbtoZtTA=Fvk){q}9Os~6mQ zKB80VI_&6iSq`LnK7*kfHZoeX6?WE}8yjuDn=2#JG$+;-TOA1%^=DnXx%w{b=w}tS zQbU3XxtOI8E(!%`64r2`zog;5<0b4i)xBmGP^jiDZ2%HNSxIf3@wKs~uk4%3Mxz;~ zts_S~E4>W+YwI<-*-$U8*^HKDEa8oLbmqGg?3vewnaNg%Mm)W=)lcC_J+1ov^u*N3 zXJ?!BrH-+wGYziJq2Y#vyry6Z>NPgkEk+Ke`^DvNRdb>Q2Nlr#v%O@<5hbflI6EKE z9dWc0-ORk^T}jP!nkJ1imyjdVX@GrjOs%cpgA8-c&FH&$(4od#x6Y&=LiJZPINVyW z0snY$8JW@>tc2}DlrD3StQmA0Twck~@>8dSix9CyQOALcREdxoM$Sw*l!}bXKq9&r zysMWR@%OY24@e`?+#xV2bk{T^C_xSo8v2ZI=lBI*l{RciPwuE>L5@uhz@{!l)rtVlWC>)6(G)1~n=Q|S!{E9~6*fdpa*n z!()-8EpTdj=zr_Lswi;#{TxbtH$8*G=UM`I+icz7sr_SdnHXrv=?iEOF1UL+*6O;% zPw>t^kbW9X@oEXx<97%lBm-9?O_7L!DeD)Me#rwE54t~UBu9VZ zl_I1tBB~>jm@bw0Aljz8! zXBB6ATG6iByKIxs!qr%pz%wgqbg(l{65DP4#v(vqhhL{0b#0C8mq`bnqZ1OwFV z7mlZZJFMACm>h9v^2J9+^_zc1=JjL#qM5ZHaThH&n zXPTsR8(+)cj&>Un{6v*z?@VTLr{TmZ@-fY%*o2G}*G}#!bmqpoo*Ay@U!JI^Q@7gj;Kg-HIrLj4}#ec4~D2~X6vo;ghep-@&yOivYP zC19L0D`jjKy1Yi-SGPAn94(768Tcf$urAf{)1)9W58P`6MA{YG%O?|07!g9(b`8PXG1B1Sh0?HQmeJtP0M$O$hI z{5G`&9XzYhh|y@qsF1GnHN|~^ru~HVf#)lOTSrv=S@DyR$UKQk zjdEPFDz{uHM&UM;=mG!xKvp;xAGHOBo~>_=WFTmh$chpC7c`~7?36h)7$fF~Ii}8q zF|YXxH-Z?d+Q+27Rs3X9S&K3N+)OBxMHn1u(vlrUC6ckBY@@jl+mgr#KQUKo#VeFm zFwNYgv0<%~Wn}KeLeD9e1$S>jhOq&(e*I@L<=I5b(?G(zpqI*WBqf|Zge0&aoDUsC zngMRA_Kt0>La+Erl=Uv_J^p(z=!?XHpenzn$%EA`JIq#yYF?JLDMYiPfM(&Csr#f{ zdd+LJL1by?xz|D8+(fgzRs~(N1k9DSyK@LJygwaYX8dZl0W!I&c^K?7)z{2is;OkE zd$VK-(uH#AUaZrp=1z;O*n=b?QJkxu`Xsw&7yrX0?(CX=I-C#T;yi8a<{E~?vr3W> zQrpPqOW2M+AnZ&p{hqmHZU-;Q(7?- zP8L|Q0RM~sB0w1w53f&Kd*y}ofx@c z5Y6B8qGel+uT1JMot$nT1!Tim6{>oZzJXdyA+4euOLME?5Fd_85Uk%#E*ln%y{u8Q z$|?|R@Hpb~yTVK-Yr_S#%NUy7EBfYGAg>b({J|5b+j-PBpPy$Ns`PaJin4JdRfOaS zE|<HjH%NuJgsd2wOlv>~y=np%=2)$M9LS|>P)zJ+Fei5vYo_N~B0XCn+GM76 z)Xz3tg*FRVFgIl9zpESgdpWAavvVViGlU8|UFY{{gVJskg*I!ZjWyk~OW-Td4(mZ6 zB&SQreAAMqwp}rjy`HsG({l2&q5Y52<@AULVAu~rWI$UbFuZs>Sc*x+XI<+ez%$U)|a^unjpiW0l0 zj1!K0(b6$8LOjzRqQ~K&dfbMIE=TF}XFAi)$+h}5SD3lo z%%Qd>p9se=VtQG{kQ;N`sI)G^u|DN#7{aoEd zkksYP%_X$Rq08);-s6o>CGJ<}v`qs%eYf+J%DQ^2k68C%nvikRsN?$ap--f+vCS`K z#&~)f7!N^;sdUXu54gl3L=LN>FB^tuK=y2e#|hWiWUls__n@L|>xH{%8lIJTd5`w? zSwZbnS;W~DawT4OwSJVdAylbY+u5S+ZH{4hAi2&}Iv~W(UvHg(1GTZRPz`@{SOqzy z(8g&Dz=$PfRV=6FgxN~zo+G8OoPI&d-thcGVR*_^(R8COTM@bq?fDwY{}WhsQS1AK zF6R1t8!RdFmfocpJ6?9Yv~;WYi~XPgs(|>{5})j!AR!voO7y9&cMPo#80A(`za@t>cx<0;qxM@S*m(jYP)dMXr*?q0E`oL;12}VAep179uEr8c<=D zr5?A*C{eJ`z9Ee;E$8)MECqatHkbHH z&Y+ho0B$31MIB-xm&;xyaFCtg<{m~M-QDbY)fQ>Q*Xibb~8ytxZQ?QMf9!%cV zU0_X1@b4d+Pg#R!`OJ~DOrQz3@cpiGy~XSKjZQQ|^4J1puvwKeScrH8o{bscBsowomu z^f12kTvje`yEI3eEXDHJ6L+O{Jv$HVj%IKb|J{IvD*l6IG8WUgDJ*UGz z3!C%>?=dlfSJ>4U88)V+`U-!9r^@AxJBx8R;)J4Fn@`~k>8>v0M9xp90OJElWP&R5 zM#v*vtT}*Gm1^)Bv!s72T3PB0yVIjJW)H7a)ilkAvoaH?)jjb`MP>2z{%Y?}83 zUIwBKn`-MSg)=?R)1Q0z3b>dHE^)D8LFs}6ASG1|daDly_^lOSy&zIIhm*HXm1?VS=_iacG);_I9c zUQH1>i#*?oPIwBMJkzi_*>HoUe}_4o>2(SHWzqQ=;TyhAHS;Enr7!#8;sdlty&(>d zl%5cjri8`2X^Ds`jnw7>A`X|bl=U8n+3LKLy(1dAu8`g@9=5iw$R0qk)w8Vh_Dt^U zIglK}sn^)W7aB(Q>HvrX=rxB z+*L)3DiqpQ_%~|m=44LcD4-bxO3OO*LPjsh%p(k?&jvLp0py57oMH|*IMa(<|{m1(0S|x)?R-mqJ=I;_YUZA>J z62v*eSK;5w!h8J+6Z2~oyGdZ68waWfy09?4fU&m7%u~zi?YPHPgK6LDwphgaYu%0j zurtw)AYOpYKgHBrkX189mlJ`q)w-f|6>IER{5Lk97%P~a-JyCRFjejW@L>n4vt6#hq;!|m;hNE||LK3nw1{bJOy+eBJjK=QqNjI;Q6;Rp5 z&035pZDUZ#%Oa;&_7x0T<7!RW`#YBOj}F380Bq?MjjEhrvlCATPdkCTTl+2efTX$k zH&0zR1n^`C3ef~^sXzJK-)52(T}uTG%OF8yDhT76L~|^+hZ2hiSM*QA9*D5odI1>& z9kV9jC~twA5MwyOx(lsGD_ggYmztXPD`2=_V|ks_FOx!_J8!zM zTzh^cc+=VNZ&(OdN=y4Juw)@8-85lwf_#VMN!Ed(eQiRiLB2^2e`4dp286h@v@`O%_b)Y~A; zv}r6U?zs&@uD_+(_4bwoy7*uozNvp?bXFoB8?l8yG0qsm1JYzIvB_OH4_2G*IIOwT zVl%HX1562vLVcxM_RG*~w_`FbIc!(T=3>r528#%mwwMK}uEhJ()3MEby zQQjzqjWkwfI~;Fuj(Lj=Ug0y`>~C7`w&wzjK(rPw+Hpd~EvQ-ufQOiB4OMpyUKJhw zqEt~jle9d7S~LI~$6Z->J~QJ{Vdn3!c}g9}*KG^Kzr^(7VI5Gk(mHLL{itj_hG?&K4Ws0+T4gLfi3eu$N=`s36geNC?c zm!~}vG6lx9Uf^5M;bWntF<-{p^bruy~f?sk9 zcETAPQZLoJ8JzMMg<-=ju4keY@SY%Wo?u9Gx=j&dfa6LIAB|IrbORLV1-H==Z1zCM zeZcOYpm5>U2fU7V*h;%n`8 zN95QhfD994={1*<2vKLCNF)feKOGk`R#K~G=;rfq}|)s20&MCa65 zUM?xF5!&e0lF%|U!#rD@I{~OsS_?=;s_MQ_b_s=PuWdC)q|UQ&ea)DMRh5>fpQjXe z%9#*x=7{iRCtBKT#H>#v%>77|{4_slZ)XCY{s3j_r{tdpvb#|r|sbS^dU1x70$eJMU!h{Y7Kd{dl}9&vxQl6Jt1a` zHQZrWyY0?!vqf@u-fxU_@+}u(%Wm>0I#KP48tiAPYY!TdW(o|KtVI|EUB9V`CBBNaBLVih7+yMVF|GSoIQD0Jfb{ z!OXq;(>Z?O`1gap(L~bUcp>Lc@Jl-})^=6P%<~~9ywY=$iu8pJ0m*hOPzr~q`23eX zgbs;VOxxENe0UMVeN*>uCn9Gk!4siN-e>x)pIKAbQz!G)TcqIJ0`JBBaX>1-4_XO_-HCS^vr2vjv#7KltDZdyQ{tlWh4$Gm zB>|O1cBDC)yG(sbnc*@w6e%e}r*|IhpXckx&;sQCwGdKH+3oSG-2)Bf#x`@<4ETAr z0My%7RFh6ZLiZ_;X6Mu1YmXx7C$lSZ^}1h;j`EZd6@%JNUe=btBE z%s=Xmo1Ps?8G`}9+6>iaB8bgjUdXT?=trMu|4yLX^m0Dg{m7rpKNJey|EwHI+nN1e zL^>qN%5Fg)dGs4DO~uwIdXImN)QJ*Jhpj7$fq_^`{3fwpztL@WBB}OwQ#Epo-mqMO zsM$UgpFiG&d#)lzEQ{3Q;)&zTw;SzGOah-Dpm{!q7<8*)Ti_;xvV2TYXa}=faXZy? z3y?~GY@kl)>G&EvEijk9y1S`*=zBJSB1iet>0;x1Ai)*`^{pj0JMs)KAM=@UyOGtO z3y0BouW$N&TnwU6!%zS%nIrnANvZF&vB1~P5_d`x-giHuG zPJ;>XkVoghm#kZXRf>qxxEix;2;D1CC~NrbO6NBX!`&_$iXwP~P*c($EVV|669kDO zKoTLZNF4Cskh!Jz5ga9uZ`3o%7Pv`d^;a=cXI|>y;zC3rYPFLQkF*nv(r>SQvD*## z(Vo%^9g`%XwS0t#94zPq;mYGLKu4LU3;txF26?V~A0xZbU4Lmy`)>SoQX^m7fd^*E z+%{R4eN!rIk~K)M&UEzxp9dbY;_I^c} zOc{wlIrN_P(PPqi51k_$>Lt|X6A^|CGYgKAmoI#Li?;Wq%q~q*L7ehZkUrMxW67Jl zhsb~+U?33QS>eqyN{(odAkbopo=Q$Az?L+NZW>j;#~@wCDX?=L5SI|OxI~7!Pli;e zELMFcZtJY3!|=Gr2L4>z8yQ-{To>(f80*#;6`4IAiqUw`=Pg$%C?#1 z_g@hIGerILSU>=P>z{gM|DS91A4cT@PEIB^hSop!uhMo#2G;+tQSpDO_6nOnPWSLU zS;a9m^DFMXR4?*X=}d7l;nXuHk&0|m`NQn%d?8|Ab3A9l9Jh5s120ibWBdB z$5YwsK3;wvp!Kn@)Qae{ef`0#NwlRpQ}k^r>yos_Ne1;xyKLO?4)t_G4eK~wkUS2A&@_;)K0-03XGBzU+5f+uMDxC z(s8!8!RvdC#@`~fx$r)TKdLD6fWEVdEYtV#{ncT-ZMX~eI#UeQ-+H(Z43vVn%Yj9X zLdu9>o%wnWdvzA-#d6Z~vzj-}V3FQ5;axDIZ;i(95IIU=GQ4WuU{tl-{gk!5{l4_d zvvb&uE{%!iFwpymz{wh?bKr1*qzeZb5f6e6m_ozRF&zux2mlK=v_(_s^R6b5lu?_W4W3#<$zeG~Pd)^!4tzhs}-Sx$FJP>)ZGF(hVTH|C3(U zs0PO&*h_ zNA-&qZpTP$$LtIgfiCn07}XDbK#HIXdmv8zdz4TY;ifNIH-0jy(gMSByG2EF~Th#eb_TueZC` zE?3I>UTMpKQ})=C;6p!?G)M6w^u*A57bD?2X`m3X^6;&4%i_m(uGJ3Z5h`nwxM<)H z$I5m?wN>O~8`BGnZ=y^p6;0+%_0K}Dcg|K;+fEi|qoBqvHj(M&aHGqNF48~XqhtU? z^ogwBzRlOfpAJ+Rw7IED8lRbTdBdyEK$gPUpUG}j-M42xDj_&qEAQEtbs>D#dRd7Y z<&TpSZ(quQDHiCFn&0xsrz~4`4tz!CdL8m~HxZM_agu@IrBpyeL1Ft}V$HX_ZqDPm z-f89)pjuEzGdq-PRu`b1m+qBGY{zr_>{6Ss>F|xHZlJj9dt5HD$u`1*WZe)qEIuDSR)%z+|n zatVlhQ?$w#XRS7xUrFE;Y8vMGhQS5*T{ZnY=q1P?w5g$OKJ#M&e??tAmPWHMj3xhS ziGxapy?kn@$~2%ZY;M8Bc@%$pkl%Rvj!?o%agBvpQ-Q61n9kznC4ttrRNQ4%GFR5u zyv%Yo9~yxQJWJSfj z?#HY$y=O~F|2pZs22pu|_&Ajd+D(Mt!nPUG{|1nlvP`=R#kKH zO*s$r_%ss5h1YO7k0bHJ2CXN)Yd6CHn~W!R=SqkWe=&nAZu(Q1G!xgcUilM@YVei@2@a`8he z9@pM`)VB*=e7-MWgLlXlc)t;fF&-AwM{E-EX}pViFn0I0CNw2bNEnN2dj!^4(^zS3 zobUm1uQnpqk_4q{pl*n06=TfK_C>UgurKFjRXsK_LEn};=79`TB12tv6KzwSu*-C8 z;=~ohDLZylHQ|Mpx-?yql>|e=vI1Z!epyUpAcDCp4T|*RV&X`Q$0ogNwy6mFALo^@ z9=&(9txO8V@E!@6^(W0{*~CT>+-MA~vnJULBxCTUW>X5>r7*eXYUT0B6+w@lzw%n> z_VjJ<2qf|(d6jYq2(x$(ZDf!yVkfnbvNmb5c|hhZ^2TV_LBz`9w!e_V*W_(MiA7|= z&EeIIkw*+$Xd!)j8<@_<}A5;~A_>3JT*kX^@}cDoLd>Qj<`Se^wdUa(j0dp+Tl8EptwBm{9OGsdFEq zM`!pjf(Lm(`$e3FLOjqA5LnN5o!}z{ zNf}rJuZh@yUtq&ErjHeGzX4(!luV!jB&;FAP|!R_QHYw#^Z1LwTePAKJ6X&IDNO#; z)#I@Xnnzyij~C@UH~X51JCgQeF0&hTXnuoElz#m{heZRexWc0k4<>0+ClX7%0 zEBqCCld1tD9Zwkr4{?Nor19#E5-YKfB8d?qgR82-Ow2^AuNevly2*tHA|sK!ybYkX zm-sLQH72P&{vEAW6+z~O5d0qd=xW~rua~5a?ymYFSD@8&gV)E5@RNNBAj^C99+Z5Z zR@Pq55mbCQbz+Mn$d_CMW<-+?TU960agEk1J<>d>0K=pF19yN))a~4>m^G&tc*xR+yMD*S=yip-q=H zIlredHpsJV8H(32@Zxc@bX6a21dUV95Th--8pE6C&3F>pk=yv$yd6@Haw;$v4+Fcb zRwn{Qo@0`7aPa2LQOP}j9v>sjOo5Kqvn|`FLizX zB+@-u4Lw|jsvz{p^>n8Vo8H2peIqJJnMN}A)q6%$Tmig7eu^}K2 zrh$X?T|ZMsoh{6pdw1G$_T<`Ds-G=jc;qcGdK4{?dN2-XxjDNbb(7pk|3JUVCU4y; z)?LXR>f+AAu)JEiti_Zy#z5{RgsC}R(@jl%9YZ>zu~hKQ*AxbvhC378-I@{~#%Y`Z zy=a=9YpewPIC+gkEUUwtUL7|RU7=!^Aa}Mk^6uxOgRGA#JXjWLsjFUnix|Mau{hDT z7mn*z1m5g`vP(#tjT0Zy4eAY(br&!RiiXE=ZI!{sE1#^#%x^Z7t1U)b<;%Y}Q9=5v z;wpDCEZ@OE36TWT=|gxigT@VaW9BvHS05;_P(#s z8zI4XFQys}q)<`tkX$WnSarn{3e!s}4(J!=Yf>+Y>cP3f;vr63f2{|S^`_pWc)^5_!R z*(x-fuBxL51@xe!lnDBKi}Br$c$BMZ3%f2Sa6kLabiBS{pq*yj;q|k(86x`PiC{p6 z_bxCW{>Q2BA8~Ggz&0jkrcU+-$ANBsOop*ms>34K9lNYil@}jC;?cYP(m^P}nR6FV zk(M%48Z&%2Rx$A&FhOEirEhY0(dn;-k(qkTU)sFQ`+-ih+s@A8g?r8Pw+}2;35WYf zi}VO`jS`p(tc)$X$a>-#WXoW!phhatC*$}|rk>|wUU71eUJG^$c6_jwX?iSHM@6__ zvV|6%U*$sSXJu9SX?2%M^kK|}a2QJ8AhF{fuXrHZxXsI~O zGKX45!K7p*MCPEQ=gp?eu&#AW*pR{lhQR##P_*{c_DjMGL|3T3-bSJ(o$|M{ytU}> zAV>wq*uE*qFo9KvnA^@juy{x<-u*#2NvkV={Ly}ysKYB-k`K3@K#^S1Bb$8Y#0L0# z`6IkSG&|Z$ODy|VLS+y5pFJx&8tvPmMd8c9FhCyiU8~k6FwkakUd^(_ml8`rnl>JS zZV){9G*)xBqPz^LDqRwyS6w86#D^~xP4($150M)SOZRe9sn=>V#aG0Iy(_^YcPpIz8QYM-#s+n% z@Jd?xQq?Xk6=<3xSY7XYP$$yd&Spu{A#uafiIfy8gRC`o0nk{ezEDjb=q_qRAlR1d zFq^*9Gn)yTG4b}R{!+3hWQ+u3GT~8nwl2S1lpw`s0X_qpxv)g+JIkVKl${sYf_nV~B>Em>M;RlqGb5WVil(89 zs=ld@|#;dq1*vQGz=7--Br-|l) zZ%Xh@v8>B7P?~}?Cg$q9_={59l%m~O&*a6TKsCMAzG&vD>k2WDzJ6!tc!V)+oxF;h zJH;apM=wO?r_+*#;ulohuP=E>^zon}a$NnlcQ{1$SO*i=jnGVcQa^>QOILc)e6;eNTI>os=eaJ{*^DE+~jc zS}TYeOykDmJ=6O%>m`i*>&pO_S;qMySJIyP=}4E&J%#1zju$RpVAkZbEl+p%?ZP^C z*$$2b4t%a(e+%>a>d_f_<JjxI#J1x;=hPd1zFPx=6T$;;X1TD*2(edZ3f46zaAoW>L53vS_J*N8TMB|n+;LD| zC=GkQPpyDY#Am4l49chDv*gojhRj_?63&&8#doW`INATAo(qY#{q}%nf@eTIXmtU< zdB<7YWfyCmBs|c)cK>1)v&M#!yNj#4d$~pVfDWQc_ke1?fw{T1Nce_b`v|Vp5ig(H zJvRD^+ps46^hLX;=e2!2e;w9y1D@!D$c@Jc&%%%IL=+xzw55&2?darw=9g~>P z9>?Kdc$r?6c$m%x2S$sdpPl>GQZ{rC9mPS63*qjCVa?OIBj!fW zm|g?>CVfGXNjOfcyqImXR_(tXS(F{FcoNzKvG5R$IgGaxC@)i(e+$ME}vPVIhd|mx2IIE+f zM?9opQHIVgBWu)^A|RzXw!^??S!x)SZOwZaJkGjc<_}2l^eSBm!eAJG9T>EC6I_sy z?bxzDIAn&K5*mX)$RQzDA?s)-no-XF(g*yl4%+GBf`##bDXJ==AQk*xmnatI;SsLp zP9XTHq5mmS=iWu~9ES>b%Q=1aMa|ya^vj$@qz9S!ih{T8_PD%Sf_QrNKwgrXw9ldm zHRVR98*{C?_XNpJn{abA!oix_mowRMu^2lV-LPi;0+?-F(>^5#OHX-fPED zCu^l7u3E%STI}c4{J2!)9SUlGP_@!d?5W^QJXOI-Ea`hFMKjR7TluLvzC-ozCPn1`Tpy z!vlv@_Z58ILX6>nDjTp-1LlFMx~-%GA`aJvG$?8*Ihn;mH37eK**rmOEwqegf-Ccx zrIX4;{c~RK>XuTXxYo5kMiWMy)!IC{*DHG@E$hx?RwP@+wuad(P1{@%tRkyJRqD)3 zMHHHZ4boqDn>-=DgR5VlhQTpfVy182Gk;A_S8A1-;U1RR>+$62>(MUx@Nox$vTjHq z%QR=j!6Gdyb5wu7y(YUktwMuW5<@jl?m4cv4BODiT5o8qVdC0MBqGr@-YBIwnpZAY znX9(_uQjP}JJ=!~Ve9#5I~rUnN|P_3D$LqZcvBnywYhjlMSFHm`;u9GPla{5QD7(7*6Tb3Svr8;(nuAd81q$*uq6HC_&~je*Ca7hP4sJp0av{M8480wF zxASi7Qv+~@2U%Nu1Ud;s-G4CTVWIPyx!sg&8ZG0Wq zG_}i3C(6_1>q3w!EH7$Kwq8uBp2F2N7}l65mk1p*9v0&+;th=_E-W)E;w}P(j⁢ zv5o9#E7!G0XmdzfsS{efPNi`1b44~SZ4Z8fuX!I}#8g+(wxzQwUT#Xb2(tbY1+EUhGKoT@KEU9Ktl>_0 z%bjDJg;#*gtJZv!-Zs`?^}v5eKmnbjqlvnSzE@_SP|LG_PJ6CYU+6zY6>92%E+ z=j@TZf-iW4(%U{lnYxQA;7Q!b;^brF8n0D>)`q5>|WDDXLrqYU_tKN2>=#@~OE7grMnNh?UOz-O~6 z6%rHy{#h9K0AT+lDC7q4{hw^|q6*Ry;;L%Q@)Ga}$60_q%D)rv(CtS$CQbpq9|y1e zRSrN4;$Jyl{m5bZw`$8TGvb}(LpY{-cQ)fcyJv7l3S52TLXVDsphtv&aPuDk1OzCA z4A^QtC(!11`IsNx_HnSy?>EKpHJWT^wmS~hc^p^zIIh@9f6U@I2 zC=Mve{j2^)mS#U$e{@Q?SO6%LDsXz@SY+=cK_QMmXBIU)j!$ajc-zLx3V60EXJ!qC zi<%2x8Q24YN+&8U@CIlN zrZkcT9yh%LrlGS9`G)KdP(@9Eo-AQz@8GEFWcb7U=a0H^ZVbLmz{+&M7W(nXJ4sN8 zJLR7eeK(K8`2-}j(T7JsO`L!+CvbueT%izanm-^A1Dn{`1Nw`9P?cq;7no+XfC`K(GO9?O^5zNIt4M+M8LM0=7Gz8UA@Z0N+lg+cX)NfazRu z5D)~HA^(u%w^cz+@2@_#S|u>GpB+j4KzQ^&Wcl9f z&hG#bCA(Yk0D&t&aJE^xME^&E-&xGHhXn%}psEIj641H+Nl-}boj;)Zt*t(4wZ5DN z@GXF$bL=&pBq-#vkTkh>7hl%K5|3 z{`Vn9b$iR-SoGENp}bn4;fR3>9sA%X2@1L3aE9yTra;Wb#_`xWwLSLdfu+PAu+o3| zGVnpzPr=ch{uuoHjtw7+_!L_2;knQ!DuDl0R`|%jr+}jFzXtrHIKc323?JO{l&;VF z*L1+}JU7%QJOg|5|Tc|D8fN zJORAg=_vsy{ak|o);@)Yh8Lkcg@$FG3k@ep36BRa^>~UmnRPziS>Z=`Jb2x*Q#`%A zU*i3&Vg?TluO@X0O;r2Jl6LKLUOVhSqg1*qOt^|8*c7 zo(298@+r$k_wQNGHv{|$tW(T8L+4_`FQ{kEW5Jgg{yf7ey4ss_(SNKfz(N9lx&a;< je(UuV8hP?p&}TPdm1I$XmG#(RzlD&B2izSj9sl%y5~4qc diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 1af9e093..9355b415 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index 1aa94a42..f5feea6d 100755 --- a/gradlew +++ b/gradlew @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## # @@ -55,7 +57,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -84,7 +86,8 @@ done # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) -APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum diff --git a/gradlew.bat b/gradlew.bat index 6689b85b..9b42019c 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -13,6 +13,8 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem @if "%DEBUG%"=="" @echo off @rem ########################################################################## @@ -43,11 +45,11 @@ set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -57,11 +59,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail From 2ace43b5ae07e0e499b6139ec1ec58ae2cec69b8 Mon Sep 17 00:00:00 2001 From: William Hinchman Date: Sat, 5 Oct 2024 10:22:10 -0400 Subject: [PATCH 486/512] Add Next Page Functionality --- .../ripper/rippers/CoomerPartyRipper.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java index e2dd301e..5b0474bd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java @@ -1,6 +1,7 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractJSONRipper; +import com.rarchives.ripme.ripper.rippers.ArtStationRipper.URL_TYPE; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; @@ -12,6 +13,8 @@ import org.json.JSONObject; import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; @@ -37,6 +40,9 @@ public class CoomerPartyRipper extends AbstractJSONRipper { private static final String KEY_PATH = "path"; private static final String KEY_ATTACHMENTS = "attachments"; + private Integer pageCount = 0; + + // One of "onlyfans" or "fansly", but might have others in future? private final String service; @@ -95,6 +101,24 @@ public class CoomerPartyRipper extends AbstractJSONRipper { return wrapperObject; } + @Override + protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException { + pageCount = pageCount + 1; + Integer offset = 50 * pageCount; + String apiUrl = String.format("https://coomer.su/api/v1/%s/user/%s?o=%d", service, user, offset); + String jsonArrayString = Http.url(apiUrl) + .ignoreContentType() + .response() + .body(); + JSONArray jsonArray = new JSONArray(jsonArrayString); + + // Ideally we'd just return the JSONArray from here, but we have to wrap it in a JSONObject + JSONObject wrapperObject = new JSONObject(); + wrapperObject.put(KEY_WRAPPER_JSON_ARRAY, jsonArray); + return wrapperObject; + } + + @Override protected List getURLsFromJSON(JSONObject json) { // extract the array from our wrapper JSONObject From 91aeb3755391b5a2f4267dc68e8b0ff87daeb943 Mon Sep 17 00:00:00 2001 From: William Hinchman Date: Sat, 5 Oct 2024 11:10:33 -0400 Subject: [PATCH 487/512] Refactor for D.R.Y. --- .../ripper/rippers/CoomerPartyRipper.java | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java index 5b0474bd..f990ae66 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java @@ -1,7 +1,5 @@ package com.rarchives.ripme.ripper.rippers; - import com.rarchives.ripme.ripper.AbstractJSONRipper; -import com.rarchives.ripme.ripper.rippers.ArtStationRipper.URL_TYPE; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; @@ -13,7 +11,6 @@ import org.json.JSONObject; import java.io.IOException; import java.net.MalformedURLException; -import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; @@ -40,15 +37,21 @@ public class CoomerPartyRipper extends AbstractJSONRipper { private static final String KEY_PATH = "path"; private static final String KEY_ATTACHMENTS = "attachments"; + // Posts Request Endpoint + private static final String POSTS_ENDPOINT = "https://coomer.su/api/v1/%s/user/%s?o=%d"; + + // Pagination is strictly 50 posts per page, per API schema. private Integer pageCount = 0; - + private static final Integer postCount = 50; - // One of "onlyfans" or "fansly", but might have others in future? + // "Service" of the page to be ripped: Onlyfans, Fansly, Candfans private final String service; - + // Username of the page to be ripped private final String user; + + public CoomerPartyRipper(URL url) throws IOException { super(url); List pathElements = Arrays.stream(url.getPath().split("/")) @@ -86,9 +89,9 @@ public class CoomerPartyRipper extends AbstractJSONRipper { return Utils.filesystemSafe(String.format("%s_%s", service, user)); } - @Override - protected JSONObject getFirstPage() throws IOException { - String apiUrl = String.format("https://coomer.su/api/v1/%s/user/%s", service, user); + private JSONObject getJsonPostsForOffset(Integer offset) throws IOException { + String apiUrl = String.format(POSTS_ENDPOINT, service, user, offset); + String jsonArrayString = Http.url(apiUrl) .ignoreContentType() .response() @@ -102,20 +105,15 @@ public class CoomerPartyRipper extends AbstractJSONRipper { } @Override - protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException { - pageCount = pageCount + 1; - Integer offset = 50 * pageCount; - String apiUrl = String.format("https://coomer.su/api/v1/%s/user/%s?o=%d", service, user, offset); - String jsonArrayString = Http.url(apiUrl) - .ignoreContentType() - .response() - .body(); - JSONArray jsonArray = new JSONArray(jsonArrayString); + protected JSONObject getFirstPage() throws IOException { + return getJsonPostsForOffset(0); + } - // Ideally we'd just return the JSONArray from here, but we have to wrap it in a JSONObject - JSONObject wrapperObject = new JSONObject(); - wrapperObject.put(KEY_WRAPPER_JSON_ARRAY, jsonArray); - return wrapperObject; + @Override + protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException { + pageCount++; + Integer offset = postCount * pageCount; + return getJsonPostsForOffset(offset); } @@ -153,6 +151,7 @@ public class CoomerPartyRipper extends AbstractJSONRipper { } } catch (JSONException e) { /* No-op */ + LOGGER.error("Unable to Parse FileURL " + e.getMessage()); } } @@ -164,7 +163,8 @@ public class CoomerPartyRipper extends AbstractJSONRipper { pullFileUrl(attachment, results); } } catch (JSONException e) { - /* No-op */ + /* No-op */ + LOGGER.error("Unable to Parse AttachmentURL " + e.getMessage()); } } From 01e0727ab100ab4223f74c72d991b3234a976b3d Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 3 Nov 2024 13:43:06 +0100 Subject: [PATCH 488/512] update to gradle-8.10.2 and github upload-artifact v4 --- .github/workflows/gradle.yml | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 675cda3e..d88a5e96 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -45,7 +45,7 @@ jobs: - name: upload jar as asset if: matrix.upload - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: zipped-ripme-jar path: build/libs/*.jar diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 9355b415..df97d72b 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME From 298f631a1f0ad9505cb49a24d834b536f925a969 Mon Sep 17 00:00:00 2001 From: Tush-r Date: Sun, 8 Sep 2024 18:30:57 +0530 Subject: [PATCH 489/512] Fix imgur ripper regex for gallery and album --- .../com/rarchives/ripme/ripper/rippers/ImgurRipper.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index b383b97e..d93b643e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -61,6 +61,7 @@ public class ImgurRipper extends AlbumRipper { return albumType == ALBUM_TYPE.USER; } + @Override public boolean canRip(URL url) { if (!url.getHost().endsWith(DOMAIN)) { return false; @@ -74,6 +75,7 @@ public class ImgurRipper extends AlbumRipper { return true; } + @Override public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { String u = url.toExternalForm(); if (u.indexOf('#') >= 0) { @@ -85,6 +87,7 @@ public class ImgurRipper extends AlbumRipper { return new URI(u).toURL(); } + @Override public String getAlbumTitle(URL url) throws MalformedURLException { String gid = null; try { @@ -468,7 +471,7 @@ public class ImgurRipper extends AlbumRipper { Pattern p; Matcher m; - p = Pattern.compile("^https?://(www\\.|m\\.)?imgur\\.com/(a|gallery)/([a-zA-Z0-9]{5,}).*$"); + p = Pattern.compile("^https?://(?:www\\.|m\\.)?imgur\\.com/gallery/(?:(?:[a-zA-Z0-9]*/)?.*-)?([a-zA-Z0-9]+)$"); m = p.matcher(url.toExternalForm()); if (m.matches()) { // Imgur album or gallery @@ -477,7 +480,7 @@ public class ImgurRipper extends AlbumRipper { this.url = new URI("http://imgur.com/a/" + gid).toURL(); return gid; } - p = Pattern.compile("^https?://(www\\.|m\\.)?imgur\\.com/(a|gallery|t)/[a-zA-Z0-9]*/([a-zA-Z0-9]{5,}).*$"); + p = Pattern.compile("^https?://(?:www\\.|m\\.)?imgur\\.com/(?:a|t)/(?:(?:[a-zA-Z0-9]*/)?.*-)?([a-zA-Z0-9]+).*$"); m = p.matcher(url.toExternalForm()); if (m.matches()) { // Imgur album or gallery From adbc989d78b55ab88f630bfe90d6ac2375336b41 Mon Sep 17 00:00:00 2001 From: Tush-r Date: Mon, 9 Sep 2024 18:48:08 +0530 Subject: [PATCH 490/512] Replace imgur usage of album ripper with abstract html ripper --- .../ripme/ripper/rippers/ImgurRipper.java | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index d93b643e..89f0dc93 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -8,6 +8,7 @@ import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -21,12 +22,12 @@ import org.jsoup.nodes.Element; import org.jsoup.safety.Safelist; import org.jsoup.select.Elements; -import com.rarchives.ripme.ripper.AlbumRipper; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; -public class ImgurRipper extends AlbumRipper { +public class ImgurRipper extends AbstractHTMLRipper { private static final String DOMAIN = "imgur.com", HOST = "imgur"; @@ -75,6 +76,22 @@ public class ImgurRipper extends AlbumRipper { return true; } + @Override + protected String getDomain() { + return DOMAIN; + } + + @Override + protected void downloadURL(URL url, int index) { + // No-op as we override rip() method + } + + @Override + protected List getURLsFromPage(Document page) { + // No-op as we override rip() method + return Arrays.asList(); + } + @Override public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { String u = url.toExternalForm(); From 169b2ecca06c6ac26fb32e839981ef204a02e8de Mon Sep 17 00:00:00 2001 From: Tush-r Date: Mon, 9 Sep 2024 18:48:30 +0530 Subject: [PATCH 491/512] Deprecaate album ripper --- src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 27ad04e8..f73dba7a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -22,6 +22,7 @@ import java.util.Map; /**' * For ripping delicious albums off the interwebz. + * @deprecated Use AbstractHTMLRipper instead. */ public abstract class AlbumRipper extends AbstractRipper { From 4cb2ef420e6a153ba86a895a5c5f1effaba16186 Mon Sep 17 00:00:00 2001 From: Tush-r Date: Mon, 9 Sep 2024 20:06:15 +0530 Subject: [PATCH 492/512] Fix imgur ripper for single image --- .../ripme/ripper/rippers/ImgurRipper.java | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index 89f0dc93..95dbcf38 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -211,12 +211,16 @@ public class ImgurRipper extends AbstractHTMLRipper { private void ripSingleImage(URL url) throws IOException, URISyntaxException { String strUrl = url.toExternalForm(); - Document document = getDocument(strUrl); - Matcher m = getEmbeddedJsonMatcher(document); - if (m.matches()) { - JSONObject json = new JSONObject(m.group(1)).getJSONObject("image"); - addURLToDownload(extractImageUrlFromJson(json), ""); + var gid = getGID(url); + var json = getSingleImageData(String.format("https://api.imgur.com/post/v1/media/%s?include=media,adconfig,account", gid)); + var media = json.getJSONArray("media"); + if (media.length()==0) { + throw new IOException(String.format("Failed to fetch image for url %s", strUrl)); + } + if (media.length()>1) { + LOGGER.warn(String.format("Got multiple images for url %s", strUrl)); } + addURLToDownload(extractImageUrlFromJson((JSONObject)media.get(0)), ""); } private void ripAlbum(URL url) throws IOException, URISyntaxException { @@ -333,11 +337,6 @@ public class ImgurRipper extends AbstractHTMLRipper { return imgurAlbum; } - private static Matcher getEmbeddedJsonMatcher(Document doc) { - Pattern p = Pattern.compile("^.*widgetFactory.mergeConfig\\('gallery', (.*?)\\);.*$", Pattern.DOTALL); - return p.matcher(doc.body().html()); - } - private static ImgurAlbum createImgurAlbumFromJsonArray(URL url, JSONArray jsonImages) throws MalformedURLException, URISyntaxException { ImgurAlbum imgurAlbum = new ImgurAlbum(url); int imagesLength = jsonImages.length(); @@ -350,21 +349,24 @@ public class ImgurRipper extends AbstractHTMLRipper { private static URL extractImageUrlFromJson(JSONObject json) throws MalformedURLException, URISyntaxException { String ext = json.getString("ext"); + if (!ext.startsWith(".")) { + ext = "." + ext; + } if (ext.equals(".gif") && Utils.getConfigBoolean("prefer.mp4", false)) { ext = ".mp4"; } return new URI( - "http://i.imgur.com/" - + json.getString("hash") + "https://i.imgur.com/" + + json.getString("id") + ext).toURL(); } - private static Document getDocument(String strUrl) throws IOException { - return Jsoup.connect(strUrl) + private static JSONObject getSingleImageData(String strUrl) throws IOException { + return Http.url(strUrl) .userAgent(USER_AGENT) .timeout(10 * 1000) - .maxBodySize(0) - .get(); + .header("Authorization", "Client-ID " + Utils.getConfigString("imgur.client_id", "546c25a59c58ad7")) + .getJSON(); } private static Document getAlbumData(String strUrl) throws IOException { From 577e848c0cee8f6203b43da267e517206c68f484 Mon Sep 17 00:00:00 2001 From: Tush-r Date: Mon, 9 Sep 2024 20:14:50 +0530 Subject: [PATCH 493/512] Remove unsupported series of images for imgur ripper --- .../ripme/ripper/rippers/ImgurRipper.java | 49 ------------------- .../com/rarchives/ripme/utils/RipUtils.java | 16 +----- .../tst/ripper/rippers/ImgurRipperTest.java | 2 + 3 files changed, 4 insertions(+), 63 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index 95dbcf38..626133ea 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -42,7 +42,6 @@ public class ImgurRipper extends AbstractHTMLRipper { USER_ALBUM, USER_IMAGES, SINGLE_IMAGE, - SERIES_OF_IMAGES, SUBREDDIT } @@ -182,10 +181,6 @@ public class ImgurRipper extends AbstractHTMLRipper { // as it seems to cause the album to be downloaded to a subdir. ripAlbum(this.url); break; - case SERIES_OF_IMAGES: - LOGGER.info("Album type is SERIES_OF_IMAGES"); - ripAlbum(this.url); - break; case SINGLE_IMAGE: LOGGER.info("Album type is SINGLE_IMAGE"); ripSingleImage(this.url); @@ -250,39 +245,6 @@ public class ImgurRipper extends AbstractHTMLRipper { } } - public static ImgurAlbum getImgurSeries(URL url) throws IOException { - Pattern p = Pattern.compile("^.*imgur\\.com/([a-zA-Z0-9,]*).*$"); - Matcher m = p.matcher(url.toExternalForm()); - ImgurAlbum album = new ImgurAlbum(url); - if (m.matches()) { - String[] imageIds = m.group(1).split(","); - for (String imageId : imageIds) { - // TODO: Fetch image with ID imageId - LOGGER.debug("Fetching image info for ID " + imageId); - try { - JSONObject json = Http.url("https://api.imgur.com/2/image/" + imageId + ".json").getJSON(); - if (!json.has("image")) { - continue; - } - JSONObject image = json.getJSONObject("image"); - if (!image.has("links")) { - continue; - } - JSONObject links = image.getJSONObject("links"); - if (!links.has("original")) { - continue; - } - String original = links.getString("original"); - ImgurImage theImage = new ImgurImage(new URI(original).toURL()); - album.addImage(theImage); - } catch (Exception e) { - LOGGER.error("Got exception while fetching imgur ID " + imageId, e); - } - } - } - return album; - } - public static ImgurAlbum getImgurAlbum(URL url) throws IOException, URISyntaxException { String strUrl = url.toExternalForm(); if (!strUrl.contains(",")) { @@ -563,17 +525,6 @@ public class ImgurRipper extends AbstractHTMLRipper { albumType = ALBUM_TYPE.SINGLE_IMAGE; return m.group(m.groupCount()); } - p = Pattern.compile("^https?://(i\\.|www\\.|m\\.)?imgur\\.com/([a-zA-Z0-9,]{5,}).*$"); - m = p.matcher(url.toExternalForm()); - if (m.matches()) { - // Series of imgur images - albumType = ALBUM_TYPE.SERIES_OF_IMAGES; - String gid = m.group(m.groupCount()); - if (!gid.contains(",")) { - throw new MalformedURLException("Imgur image doesn't contain commas"); - } - return gid.replaceAll(",", "-"); - } throw new MalformedURLException("Unsupported imgur URL format: " + url.toExternalForm()); } diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 8067cd92..15e4128f 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -45,20 +45,8 @@ public class RipUtils { logger.error("[!] Exception while loading album " + url, e); } return result; - } - else if (url.getHost().endsWith("imgur.com") && url.toExternalForm().contains(",")) { - // Imgur image series. - try { - logger.debug("Fetching imgur series at " + url); - ImgurRipper.ImgurAlbum imgurAlbum = ImgurRipper.getImgurSeries(url); - for (ImgurRipper.ImgurImage imgurImage : imgurAlbum.images) { - logger.debug("Got imgur image: " + imgurImage.url); - result.add(imgurImage.url); - } - } catch (IOException e) { - logger.error("[!] Exception while loading album " + url, e); - } - } else if (url.getHost().endsWith("i.imgur.com") && url.toExternalForm().contains("gifv")) { + } + else if (url.getHost().endsWith("i.imgur.com") && url.toExternalForm().contains("gifv")) { // links to imgur gifvs try { result.add(new URI(url.toExternalForm().replaceAll(".gifv", ".mp4")).toURL()); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java index 5eeb077f..6e1c3714 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java @@ -27,6 +27,8 @@ public class ImgurRipperTest extends RippersTest { failURLs.add(new URI("http://i.imgur.com/").toURL()); failURLs.add(new URI("http://imgur.com/image.jpg").toURL()); failURLs.add(new URI("http://i.imgur.com/image.jpg").toURL()); + // Imgur seems not to support URLs with lists of images anymore. + failURLs.add(new URI("http://imgur.com/758qD43,C6iVJex,bP7flAu,J3l85Ri,1U7fhu5,MbuAUCM,JF4vOXQ").toURL()); for (URL url : failURLs) { try { new ImgurRipper(url); From c88018c25b385b558162eedac2a03d8f296993f1 Mon Sep 17 00:00:00 2001 From: Tush-r Date: Wed, 11 Sep 2024 17:22:01 +0530 Subject: [PATCH 494/512] Fix imgur not prefixing album images --- .../com/rarchives/ripme/ripper/rippers/ImgurRipper.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index 626133ea..a11442a6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -231,16 +231,18 @@ public class ImgurRipper extends AbstractHTMLRipper { stopCheck(); Path saveAs = workingDir.toPath(); if (subdirectory != null && !subdirectory.equals("")) { - saveAs.resolve(subdirectory); + saveAs = saveAs.resolve(subdirectory); } if (!Files.exists(saveAs)) { Files.createDirectory(saveAs); } index += 1; + var imgPath = imgurImage.getSaveAs().replaceAll("\\?\\d", ""); if (Utils.getConfigBoolean("download.save_order", true)) { - saveAs.resolve(String.format("%03d_", index)); + saveAs = saveAs.resolve(String.format("%03d_%s", index, imgPath)); + } else { + saveAs = saveAs.resolve(imgPath); } - saveAs = saveAs.resolve(imgurImage.getSaveAs().replaceAll("\\?\\d", "")); addURLToDownload(imgurImage.url, saveAs); } } From 2fc3853df502f5327f75ca874fc1022dc8f28543 Mon Sep 17 00:00:00 2001 From: Tush-r Date: Wed, 11 Sep 2024 17:34:28 +0530 Subject: [PATCH 495/512] Fix imgur ripper for user account --- .../ripme/ripper/rippers/ImgurRipper.java | 76 +++++++++++++++---- 1 file changed, 61 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index a11442a6..ddf34cef 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -343,29 +343,66 @@ public class ImgurRipper extends AbstractHTMLRipper { .get(); } + private static JSONObject getUserData(String userUrl) throws IOException { + return Http.url(userUrl) + .userAgent(USER_AGENT) + .timeout(10 * 1000) + .header("Authorization", "Client-ID " + Utils.getConfigString("imgur.client_id", "546c25a59c58ad7")) + .getJSON(); + } + /** * Rips all albums in an imgur user's account. * @param url - * URL to imgur user account (http://username.imgur.com) + * URL to imgur user account (http://username.imgur.com | https://imgur.com/user/username) */ private void ripUserAccount(URL url) throws IOException, URISyntaxException { + int cPage = -1, cImage = 0; + String apiUrl = "https://api.imgur.com/3/account/%s/submissions/%d/newest?album_previews=1"; + // Strip 'user_' from username + var username = getGID(url).replace("user_", ""); LOGGER.info("Retrieving " + url); sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm()); - Document doc = Http.url(url).get(); - for (Element album : doc.select("div.cover a")) { - stopCheck(); - if (!album.hasAttr("href") - || !album.attr("href").contains("imgur.com/a/")) { - continue; + + while (true) { + cPage += 1; + var pageUrl = String.format(apiUrl, username, cPage); + var json = getUserData(pageUrl); + var success = json.getBoolean("success"); + var status = json.getInt("status"); + if (!success || status!=200) { + throw new IOException(String.format("Unexpected status code %d for url %s and page %d", status, url, cPage)); } - String albumID = album.attr("href").substring(album.attr("href").lastIndexOf('/') + 1); - URL albumURL = new URI("http:" + album.attr("href") + "/noscript").toURL(); - try { - ripAlbum(albumURL, albumID); - Thread.sleep(SLEEP_BETWEEN_ALBUMS * 1000L); - } catch (Exception e) { - LOGGER.error("Error while ripping album: " + e.getMessage(), e); + var data = json.getJSONArray("data"); + if (data.isEmpty()) { + // Data array is empty for pages beyond the last page + break; + } + for (int i = 0; i < data.length(); i++) { + cImage += 1; + String prefixOrSubdir = ""; + if (Utils.getConfigBoolean("download.save_order", true)) { + prefixOrSubdir = String.format("%03d_", cImage); + } + var d = (JSONObject)data.get(i); + var l = d.getString("link"); + if (d.getBoolean("is_album")) { + // For album links with multiple images create a prefixed folder with album id + prefixOrSubdir += d.getString("id"); + ripAlbum(new URI(l).toURL(), prefixOrSubdir); + try { + Thread.sleep(SLEEP_BETWEEN_ALBUMS * 1000L); + } catch (InterruptedException e) { + LOGGER.error(String.format("Error! Interrupted ripping album %s for user account %s", l, username), e); + } + } else { + // For direct links + if (d.has("mp4") && Utils.getConfigBoolean("prefer.mp4", false)) { + l = d.getString("mp4"); + } + addURLToDownload(new URI(l).toURL(), prefixOrSubdir); + } } } } @@ -463,6 +500,7 @@ public class ImgurRipper extends AbstractHTMLRipper { this.url = new URI("http://imgur.com/a/" + gid).toURL(); return gid; } + // Match urls with path /a p = Pattern.compile("^https?://(?:www\\.|m\\.)?imgur\\.com/(?:a|t)/(?:(?:[a-zA-Z0-9]*/)?.*-)?([a-zA-Z0-9]+).*$"); m = p.matcher(url.toExternalForm()); if (m.matches()) { @@ -472,7 +510,7 @@ public class ImgurRipper extends AbstractHTMLRipper { this.url = new URI("http://imgur.com/a/" + gid).toURL(); return gid; } - p = Pattern.compile("^https?://([a-zA-Z0-9\\-]{3,})\\.imgur\\.com/?$"); + p = Pattern.compile("^https?://([a-zA-Z0-9\\-]{4,})\\.imgur\\.com/?$"); m = p.matcher(url.toExternalForm()); if (m.matches()) { // Root imgur account @@ -483,6 +521,14 @@ public class ImgurRipper extends AbstractHTMLRipper { albumType = ALBUM_TYPE.USER; return "user_" + gid; } + // Pattern for new imgur user url https://imgur.com/user/username + p = Pattern.compile("^https?://(?:www\\.|m\\.)?imgur\\.com/user/([a-zA-Z0-9]+).*$"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + String gid = m.group(1); + albumType = ALBUM_TYPE.USER; + return "user_" + gid; + } p = Pattern.compile("^https?://([a-zA-Z0-9\\-]{3,})\\.imgur\\.com/all.*$"); m = p.matcher(url.toExternalForm()); if (m.matches()) { From d439ff0b3ed95076b6a994618f771acd12ceba07 Mon Sep 17 00:00:00 2001 From: Tush-r Date: Thu, 12 Sep 2024 15:33:50 +0530 Subject: [PATCH 496/512] Fix imgur ripper tests --- .../tst/ripper/rippers/ImgurRipperTest.java | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java index 6e1c3714..86dc334b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java @@ -5,8 +5,6 @@ import com.rarchives.ripme.ripper.rippers.ImgurRipper.ImgurAlbum; import com.rarchives.ripme.utils.RipUtils; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -40,23 +38,23 @@ public class ImgurRipperTest extends RippersTest { } @Test - @Tag("flaky") public void testImgurAlbums() throws IOException, URISyntaxException { List contentURLs = new ArrayList<>(); // URLs that should return more than 1 image - //contentURLs.add(new URI("http://imgur.com/a/dS9OQ#0").toURL()); // Horizontal layout - //contentURLs.add(new URI("http://imgur.com/a/YpsW9#0").toURL()); // Grid layout - contentURLs.add(new URI("http://imgur.com/a/WxG6f/layout/vertical#0").toURL()); - contentURLs.add(new URI("http://imgur.com/a/WxG6f/layout/horizontal#0").toURL()); - contentURLs.add(new URI("http://imgur.com/a/WxG6f/layout/grid#0").toURL()); - contentURLs.add(new URI("http://imgur.com/gallery/FmP2o").toURL()); // Gallery URL - // Imgur seems not to support URLs with lists of images anymore. - // contentURLs.add(new - // URL("http://imgur.com/758qD43,C6iVJex,bP7flAu,J3l85Ri,1U7fhu5,MbuAUCM,JF4vOXQ")); + contentURLs.add(new URI("http://imgur.com/gallery/FmP2o").toURL()); + // URLs with /gallery path + contentURLs.add(new URI("http://imgur.com/gallery/nAl13J6").toURL()); + contentURLs.add(new URI("https://imgur.com/gallery/another-brendan-fraser-reaction-from-bedazzled-intergalactic-quality-nAl13J6").toURL()); + // URLs with /a path + contentURLs.add(new URI("http://imgur.com/a/G058j5F").toURL()); + contentURLs.add(new URI("https://imgur.com/a/thanks-batman-G058j5F").toURL()); + contentURLs.add(new URI("https://imgur.com/a/thanks-batman-G058j5F/layout/grid#0").toURL()); + contentURLs.add(new URI("https://imgur.com/a/G058j5F/layout/grid#0").toURL()); + contentURLs.add(new URI("https://imgur.com/a/G058j5F/layout/horizontal#0").toURL()); // Sometimes hangs up // contentURLs.add(new URI("http://imgur.com/r/nsfw_oc/top/all").toURL()); - // contentURLs.add(new URI("http://imgur.com/a/bXQpH").toURL()); // Album with - // titles/descriptions + // Album with titles/descriptions + contentURLs.add(new URI("http://imgur.com/a/bXQpH").toURL()); for (URL url : contentURLs) { ImgurRipper ripper = new ImgurRipper(url); testRipper(ripper); @@ -64,7 +62,21 @@ public class ImgurRipperTest extends RippersTest { } @Test - @Disabled("test or ripper broken") + public void testImgurUserAccount() throws IOException, URISyntaxException { + List contentURLs = new ArrayList<>(); + // URL with albums + contentURLs.add("https://RockStarBrew.imgur.com"); + // New URL format + contentURLs.add("https://imgur.com/user/RockStarBrew/"); + // And URL with images + contentURLs.add("https://imgur.com/user/counter2strike"); + for (var url : contentURLs) { + ImgurRipper ripper = new ImgurRipper(new URI(url).toURL()); + testRipper(ripper); + } + } + + @Test public void testImgurSingleImage() throws IOException, URISyntaxException { List contentURLs = new ArrayList<>(); contentURLs.add(new URI("http://imgur.com/qbfcLyG").toURL()); // Single image URL @@ -91,8 +103,8 @@ public class ImgurRipperTest extends RippersTest { @Test public void testImgurVideoFromGetFilesFromURL() throws Exception { - List urls = RipUtils.getFilesFromURL(new URI("https://i.imgur.com/4TtwxRN.gifv").toURL()); - Assertions.assertEquals("https://i.imgur.com/4TtwxRN.mp4", urls.get(0).toExternalForm()); + List urls = RipUtils.getFilesFromURL(new URI("https://i.imgur.com/7qoW0Mo.gifv").toURL()); + Assertions.assertEquals("https://i.imgur.com/7qoW0Mo.mp4", urls.get(0).toExternalForm()); } /* From e7a531b8b641894d94b7dcf69d19bbdaa610874d Mon Sep 17 00:00:00 2001 From: Tush-r Date: Thu, 12 Sep 2024 16:46:46 +0530 Subject: [PATCH 497/512] Fix imgur url scheme and remove json from logs --- .../rarchives/ripme/ripper/rippers/ImgurRipper.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index ddf34cef..4904ac60 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -255,8 +255,6 @@ public class ImgurRipper extends AbstractHTMLRipper { LOGGER.info(" Retrieving " + strUrl); Document doc = getAlbumData("https://api.imgur.com/3/album/" + strUrl.split("/a/")[1]); // Try to use embedded JSON to retrieve images - LOGGER.info(Jsoup.clean(doc.body().toString(), Safelist.none())); - try { JSONObject json = new JSONObject(Jsoup.clean(doc.body().toString(), Safelist.none())); JSONArray jsonImages = json.getJSONObject("data").getJSONArray("images"); @@ -427,7 +425,7 @@ public class ImgurRipper extends AbstractHTMLRipper { for (int i = 0; i < images.length(); i++) { imagesFound++; JSONObject image = images.getJSONObject(i); - String imageUrl = "http://i.imgur.com/" + image.getString("hash") + image.getString("ext"); + String imageUrl = "https://i.imgur.com/" + image.getString("hash") + image.getString("ext"); String prefix = ""; if (Utils.getConfigBoolean("download.save_order", true)) { prefix = String.format("%03d_", imagesFound); @@ -497,7 +495,7 @@ public class ImgurRipper extends AbstractHTMLRipper { // Imgur album or gallery albumType = ALBUM_TYPE.ALBUM; String gid = m.group(m.groupCount()); - this.url = new URI("http://imgur.com/a/" + gid).toURL(); + this.url = new URI("https://imgur.com/a/" + gid).toURL(); return gid; } // Match urls with path /a @@ -507,7 +505,7 @@ public class ImgurRipper extends AbstractHTMLRipper { // Imgur album or gallery albumType = ALBUM_TYPE.ALBUM; String gid = m.group(m.groupCount()); - this.url = new URI("http://imgur.com/a/" + gid).toURL(); + this.url = new URI("https://imgur.com/a/" + gid).toURL(); return gid; } p = Pattern.compile("^https?://([a-zA-Z0-9\\-]{4,})\\.imgur\\.com/?$"); @@ -563,7 +561,7 @@ public class ImgurRipper extends AbstractHTMLRipper { albumType = ALBUM_TYPE.ALBUM; String subreddit = m.group(m.groupCount() - 1); String gid = m.group(m.groupCount()); - this.url = new URI("http://imgur.com/r/" + subreddit + "/" + gid).toURL(); + this.url = new URI("https://imgur.com/r/" + subreddit + "/" + gid).toURL(); return "r_" + subreddit + "_" + gid; } p = Pattern.compile("^https?://(i\\.|www\\.|m\\.)?imgur\\.com/([a-zA-Z0-9]{5,})$"); From 5d7897d2aca6f975f5490fbfee9f83272c8756a8 Mon Sep 17 00:00:00 2001 From: Tush-r Date: Thu, 12 Sep 2024 16:48:12 +0530 Subject: [PATCH 498/512] Remove 96 character limit for pasted urls --- .../rarchives/ripme/uiUtils/ContextActionProtections.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java b/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java index e247926c..9237fea9 100644 --- a/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java +++ b/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java @@ -17,10 +17,11 @@ public class ContextActionProtections { try { String clipboardContent = (String) transferable.getTransferData(DataFlavor.stringFlavor); + // TODO check if commenting this causes regression // Limit the pasted content to 96 characters - if (clipboardContent.length() > 96) { - clipboardContent = clipboardContent.substring(0, 96); - } + // if (clipboardContent.length() > 96) { + // clipboardContent = clipboardContent.substring(0, 96); + // } // Set the text in the JTextField textComponent.setText(clipboardContent); } catch (UnsupportedFlavorException | IOException unable_to_modify_text_on_paste) { From afe0728671828661da877650f6adc5b09203b955 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 3 Nov 2024 13:47:34 +0100 Subject: [PATCH 499/512] test with java-23 --- .github/workflows/gradle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index d88a5e96..d1d140d1 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - java: [22] + java: [23] include: # test old java on one os only, upload from ubuntu java-17 - os: ubuntu-latest java: 21 From b0214bca0533893cf871b5f190f07e1d21de900b Mon Sep 17 00:00:00 2001 From: soloturn Date: Thu, 12 Dec 2024 05:14:49 +0100 Subject: [PATCH 500/512] imgur 100, ehentai flaky tests --- build.gradle.kts | 2 +- .../rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java | 4 +++- .../rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index e4c08af6..f1f4e9aa 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -43,7 +43,7 @@ version = "1.7.94" description = "ripme" jacoco { - toolVersion = "0.8.11" + toolVersion = "0.8.12" } jgitver { diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java index 31fee74e..dbd107bb 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java @@ -9,10 +9,12 @@ import java.util.List; import com.rarchives.ripme.ripper.rippers.EHentaiRipper; import com.rarchives.ripme.utils.RipUtils; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class EhentaiRipperTest extends RippersTest { @Test + @Tag("flaky") public void testEHentaiAlbum() throws IOException, URISyntaxException { EHentaiRipper ripper = new EHentaiRipper(new URI("https://e-hentai.org/g/1144492/e823bdf9a5/").toURL()); testRipper(ripper); @@ -34,4 +36,4 @@ public class EhentaiRipperTest extends RippersTest { blacklistedTag = RipUtils.checkTags(tags2, tagsOnPage); Assertions.assertEquals("midnight on mars", blacklistedTag); } -} \ No newline at end of file +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java index 86dc334b..3dacc1bb 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java @@ -5,6 +5,7 @@ import com.rarchives.ripme.ripper.rippers.ImgurRipper.ImgurAlbum; import com.rarchives.ripme.utils.RipUtils; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -95,6 +96,7 @@ public class ImgurRipperTest extends RippersTest { } @Test + @Tag("flaky") public void testImgurAlbumWithMoreThan100Pictures() throws IOException, URISyntaxException { ImgurAlbum album = ImgurRipper.getImgurAlbum(new URI("https://imgur.com/a/HX3JSrD").toURL()); Assertions.assertTrue(album.images.size() >= 100, From 321a4bfba197495719107afa9e2c76979368e56c Mon Sep 17 00:00:00 2001 From: joroto Date: Tue, 19 Nov 2024 20:57:09 +0300 Subject: [PATCH 501/512] App does not open set to most recently used language. --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 8b547d7d..030eaedb 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -570,7 +570,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { configLogLevelCombobox = new JComboBox<>( new String[] { "Log level: Error", "Log level: Warn", "Log level: Info", "Log level: Debug" }); configSelectLangComboBox = new JComboBox<>(Utils.getSupportedLanguages()); - configSelectLangComboBox.setSelectedItem(Utils.getSelectedLanguage()); + configSelectLangComboBox.setSelectedItem(Utils.getConfigString("lang", Utils.getSelectedLanguage())); configLogLevelCombobox.setSelectedItem(Utils.getConfigString("log.level", "Log level: Debug")); setLogLevel(configLogLevelCombobox.getSelectedItem().toString()); configSaveDirLabel = new JLabel(); From e86161dd35fad6519ee2d1bd2526e2dc8fff110d Mon Sep 17 00:00:00 2001 From: joroto Date: Tue, 19 Nov 2024 20:10:18 +0300 Subject: [PATCH 502/512] E-hentai ripper not working fix. --- .../java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java index 81f09aa4..cccfeb09 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java @@ -177,7 +177,7 @@ public class EHentaiRipper extends AbstractHTMLRipper { @Override public List getURLsFromPage(Document page) { List imageURLs = new ArrayList<>(); - Elements thumbs = page.select("#gdt > .gdtm a"); + Elements thumbs = page.select("#gdt > a"); // Iterate over images on page for (Element thumb : thumbs) { imageURLs.add(thumb.attr("href")); From ca96ce8849258868e3a0cebfd86f93e4e9b92028 Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 13 Dec 2024 04:05:15 +0100 Subject: [PATCH 503/512] ehentai test switch on, listal switch off --- .../ripme/tst/ripper/rippers/EhentaiRipperTest.java | 5 +---- .../rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java | 4 +++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java index dbd107bb..a8393daf 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java @@ -8,13 +8,10 @@ import java.util.List; import com.rarchives.ripme.ripper.rippers.EHentaiRipper; import com.rarchives.ripme.utils.RipUtils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; public class EhentaiRipperTest extends RippersTest { @Test - @Tag("flaky") public void testEHentaiAlbum() throws IOException, URISyntaxException { EHentaiRipper ripper = new EHentaiRipper(new URI("https://e-hentai.org/g/1144492/e823bdf9a5/").toURL()); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java index 4516f2c8..259114ba 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ListalRipperTest.java @@ -5,7 +5,7 @@ import java.net.URI; import java.net.URISyntaxException; import com.rarchives.ripme.ripper.rippers.ListalRipper; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; public class ListalRipperTest extends RippersTest { @@ -13,6 +13,7 @@ public class ListalRipperTest extends RippersTest { * Test for list type url. */ @Test + @Tag("flaky") public void testPictures() throws IOException, URISyntaxException { ListalRipper ripper = new ListalRipper(new URI("https://www.listal.com/emma-stone_iii/pictures").toURL()); @@ -23,6 +24,7 @@ public class ListalRipperTest extends RippersTest { * Test for list type url. */ @Test + @Tag("flaky") public void testRipListType() throws IOException, URISyntaxException { ListalRipper ripper = new ListalRipper(new URI("https://www.listal.com/list/evolution-emma-stone").toURL()); From 8afe86fc5d26abb57474acdbeb8df20fa2f454ab Mon Sep 17 00:00:00 2001 From: soloturn Date: Fri, 13 Dec 2024 04:41:46 +0100 Subject: [PATCH 504/512] release 2.1.11-20-ca96ce88 --- ripme.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ripme.json b/ripme.json index c3a90ca0..c3de8d8a 100644 --- a/ripme.json +++ b/ripme.json @@ -1,7 +1,8 @@ { - "latestVersion": "2.1.10-21-c94a9543", - "currentHash": "782ffec29bd14cfde6d714fa6f76980b3fd7cf96723b1121976134a6a5057e68", + "latestVersion": "2.1.11-20-ca96ce88", + "currentHash": "f882a4d8a73512f7c658b87049bd9b85d9ef50c4241b6c17bd643e122573e732", "changeList": [ + "2.1.11-20-ca96ce88, Commer.party next page, Imgur, E-hentai fixed, set recent language.", "2.1.10-21-c94a9543, Imagebam, Unify colons in UI, Motherless, right click menu, rgif fixed", "2.1.9-7-22e915df, HistoryMenuMouseListener right click menu, Imagefap retry logic for getFullSizedImage(), EightmusesRipper fixed", "2.1.8-1-f5153de8: jpg3 add, java-21 adjustments.", From 9a6de2509d7cfd200c86219b42682493b51c0382 Mon Sep 17 00:00:00 2001 From: ipkpjersi <33754783+ipkpjersi@users.noreply.github.com> Date: Sat, 14 Dec 2024 17:18:04 -0500 Subject: [PATCH 505/512] Removed unnecessary new Java syntax This line breaks Java 17 compatibility, so I fixed it. --- .../rarchives/ripme/ripper/rippers/video/VidearnRipper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java index 3fbb6375..707fa03f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java @@ -57,8 +57,8 @@ public class VidearnRipper extends VideoRipper { if (mp4s.isEmpty()) { throw new IOException("Could not find files at " + url); } - String vidUrl = mp4s.getFirst(); + String vidUrl = mp4s.get(0); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); waitForThreads(); } -} \ No newline at end of file +} From ec728ab5e4a6fd5081a31c0a03633cf29cb09957 Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 29 Dec 2024 11:33:51 +0100 Subject: [PATCH 506/512] use processbuilder for ripme update on windows, exec is deprecated. --- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 128eabba..18eb5512 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -296,10 +296,9 @@ public class UpdateUtils { Runtime.getRuntime().addShutdownHook(new Thread(() -> { try { logger.info("Executing: " + batchFile); - Runtime.getRuntime().exec(String.valueOf(batchFile)); + ProcessBuilder processBuilder = new ProcessBuilder(String.valueOf(batchFile)); + processBuilder.start(); } catch (IOException e) { - // TODO implement proper stack trace handling this is really just intented as a - // placeholder until you implement proper error handling e.printStackTrace(); } })); From 64872194696dca9d4ddab19d397865f84f9900da Mon Sep 17 00:00:00 2001 From: soloturn Date: Sun, 29 Dec 2024 11:52:46 +0100 Subject: [PATCH 507/512] mark albumripper deprecated --- src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index f73dba7a..0f3a1e7a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -24,6 +24,7 @@ import java.util.Map; * For ripping delicious albums off the interwebz. * @deprecated Use AbstractHTMLRipper instead. */ +@Deprecated public abstract class AlbumRipper extends AbstractRipper { private Map itemsPending = Collections.synchronizedMap(new HashMap()); From 255e388af92ce0a896d08322de43499f55a7d5ec Mon Sep 17 00:00:00 2001 From: joroto <47276635+joroto@users.noreply.github.com> Date: Wed, 1 Jan 2025 09:44:07 +0200 Subject: [PATCH 508/512] Ignore SSL verification option added (#175) * add configuration to ignore SS verification --- .../ripme/ripper/AbstractHTMLRipper.java | 5 +++ .../com/rarchives/ripme/ui/MainWindow.java | 11 ++++- .../java/com/rarchives/ripme/utils/Http.java | 42 +++++++++++++++++++ src/main/resources/LabelsBundle.properties | 1 + 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 8c26c903..e7b646e5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -36,6 +36,11 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { protected AbstractHTMLRipper(URL url) throws IOException { super(url); + if(Utils.getConfigBoolean("ssl.verify.off",false)){ + Http.SSLVerifyOff(); + }else { + Http.undoSSLVerifyOff(); + } } protected abstract String getDomain(); diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 030eaedb..13e05fed 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -95,6 +95,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static JCheckBox configAutoupdateCheckbox; private static JComboBox configLogLevelCombobox; private static JCheckBox configURLHistoryCheckbox; + private static JCheckBox configSSLVerifyOff; private static JCheckBox configPlaySound; private static JCheckBox configSaveOrderCheckbox; private static JCheckBox configShowPopup; @@ -212,6 +213,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { Utils.setConfigBoolean("descriptions.save", configSaveDescriptions.isSelected()); Utils.setConfigBoolean("prefer.mp4", configPreferMp4.isSelected()); Utils.setConfigBoolean("remember.url_history", configURLHistoryCheckbox.isSelected()); + Utils.setConfigBoolean("ssl.verify.off", configSSLVerifyOff.isSelected()); Utils.setConfigString("lang", configSelectLangComboBox.getSelectedItem().toString()); saveWindowPosition(mainFrame); saveHistory(); @@ -565,6 +567,8 @@ public final class MainWindow implements Runnable, RipStatusHandler { true); configURLHistoryCheckbox = addNewCheckbox(Utils.getLocalizedString("remember.url.history"), "remember.url_history", true); + configSSLVerifyOff = addNewCheckbox(Utils.getLocalizedString("ssl.verify.off"), + "ssl.verify.off", false); configUrlFileChooserButton = new JButton(Utils.getLocalizedString("download.url.list")); configLogLevelCombobox = new JComboBox<>( @@ -599,6 +603,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { addItemToConfigGridBagConstraints(gbc, idx++, configClipboardAutorip, configSaveAlbumTitles); addItemToConfigGridBagConstraints(gbc, idx++, configSaveDescriptions, configPreferMp4); addItemToConfigGridBagConstraints(gbc, idx++, configWindowPosition, configURLHistoryCheckbox); + addItemToConfigGridBagConstraints(gbc, idx++, configSSLVerifyOff, configSSLVerifyOff); addItemToConfigGridBagConstraints(gbc, idx++, configSelectLangComboBox, configUrlFileChooserButton); addItemToConfigGridBagConstraints(gbc, idx++, configSaveDirLabel, configSaveDirButton); @@ -738,6 +743,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { configPreferMp4.setText(Utils.getLocalizedString("prefer.mp4.over.gif")); configWindowPosition.setText(Utils.getLocalizedString("restore.window.position")); configURLHistoryCheckbox.setText(Utils.getLocalizedString("remember.url.history")); + configSSLVerifyOff.setText(Utils.getLocalizedString("ssl.verify.off")); optionLog.setText(Utils.getLocalizedString("Log")); optionHistory.setText(Utils.getLocalizedString("History")); optionQueue.setText(Utils.getLocalizedString("queue")); @@ -1012,6 +1018,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { addCheckboxListener(configSaveLogs, "log.save"); addCheckboxListener(configSaveURLsOnly, "urls_only.save"); addCheckboxListener(configURLHistoryCheckbox, "remember.url_history"); + addCheckboxListener(configSSLVerifyOff, "ssl.verify.off"); addCheckboxListener(configSaveAlbumTitles, "album_titles.save"); addCheckboxListener(configSaveDescriptions, "descriptions.save"); addCheckboxListener(configPreferMp4, "prefer.mp4"); @@ -1528,8 +1535,8 @@ public final class MainWindow implements Runnable, RipStatusHandler { } /* * content key %path% the path to the album folder %url% is the album url - * - * + * + * */ if (Utils.getConfigBoolean("enable.finish.command", false)) { try { diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index 374f32e7..a1705f5a 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -13,11 +13,14 @@ import org.jsoup.HttpStatusException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; +import javax.net.ssl.*; import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; import java.util.HashMap; import java.util.Map; @@ -230,4 +233,43 @@ public class Http { } throw new IOException("Failed to load " + url + " after " + this.retries + " attempts", lastException); } + + public static void SSLVerifyOff() { + try { + TrustManager[] trustAllCerts = new TrustManager[]{ + new X509TrustManager() { + public X509Certificate[] getAcceptedIssuers() { + return null; + } + + public void checkClientTrusted(X509Certificate[] certs, String authType) { + } + + public void checkServerTrusted(X509Certificate[] certs, String authType) { + } + } + }; + SSLContext sslContext = SSLContext.getInstance("SSL"); + sslContext.init(null, trustAllCerts, new SecureRandom()); + HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory()); + HostnameVerifier allHostsValid = (hostname, session) -> true; + HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid); + } catch (Exception e) { + logger.error("ignoreSSLVerification() failed."); + logger.error(e.getMessage()); + } + } + + public static void undoSSLVerifyOff() { + try { + // Reset to the default SSL socket factory and hostname verifier + SSLContext sslContext = SSLContext.getInstance("SSL"); + sslContext.init(null, null, new SecureRandom()); + HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory()); + HttpsURLConnection.setDefaultHostnameVerifier(HttpsURLConnection.getDefaultHostnameVerifier()); + } catch (Exception e) { + logger.error("undoSSLVerificationIgnore() failed."); + logger.error(e.getMessage()); + } + } } diff --git a/src/main/resources/LabelsBundle.properties b/src/main/resources/LabelsBundle.properties index 983086c2..6a48b245 100644 --- a/src/main/resources/LabelsBundle.properties +++ b/src/main/resources/LabelsBundle.properties @@ -26,6 +26,7 @@ save.descriptions = Save descriptions prefer.mp4.over.gif = Prefer MP4 over GIF restore.window.position = Restore window position remember.url.history = Remember URL history +ssl.verify.off = SSL verify off loading.history.from = Loading history from # Queue keys From 8499c5f3011c4009d214184617a376e3d96368cd Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 1 Jan 2025 09:35:36 +0100 Subject: [PATCH 509/512] drop ripmeapp2, use ripmeapp repo --- src/main/java/com/rarchives/ripme/ui/UpdateUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java index 18eb5512..d5f82a36 100644 --- a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java @@ -30,7 +30,7 @@ public class UpdateUtils { private static final Logger logger = LogManager.getLogger(UpdateUtils.class); // do not update the default version without adjusting the unit test. the real version comes from METAINF.MF private static final String DEFAULT_VERSION = "1.7.94-10-b6345398"; - private static final String REPO_NAME = "ripmeapp2/ripme"; + private static final String REPO_NAME = "ripmeapp/ripme"; private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/main/ripme.json"; private static final Path newFile = Paths.get("ripme.jar.new"); private static Path mainFile; @@ -52,7 +52,7 @@ public class UpdateUtils { return "https://github.com/" + REPO_NAME + "/releases/download/" - + latestVersion.substring(0, latestVersion.indexOf("-")) + + latestVersion + "/ripme-" + latestVersion + ".jar"; } From d0b97acda81e4ad3cbbd7c7b36c1e2d48262fa00 Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 1 Jan 2025 09:39:58 +0100 Subject: [PATCH 510/512] hentaidude flaky, or not existing. --- .../ripme/tst/ripper/rippers/HentaidudeRipperTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaidudeRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaidudeRipperTest.java index 0283f9b7..e2cbd754 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaidudeRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaidudeRipperTest.java @@ -1,6 +1,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.HentaidudeRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -10,6 +11,7 @@ import java.net.URISyntaxException; public class HentaidudeRipperTest extends RippersTest{ @Test + @Tag("flaky") public void testRip() throws IOException, URISyntaxException { HentaidudeRipper ripper = new HentaidudeRipper(new URI("https://hentaidude.com/girlfriends-4ever-dlc-2/").toURL()); testRipper(ripper); From f3fca3887fbf1024f84147ba240793d62ae14676 Mon Sep 17 00:00:00 2001 From: soloturn Date: Wed, 1 Jan 2025 11:03:07 +0100 Subject: [PATCH 511/512] release 2.1.12-7-d0b97acd, switch back to ripme gihub project --- ripme.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ripme.json b/ripme.json index c3de8d8a..c95f9381 100644 --- a/ripme.json +++ b/ripme.json @@ -1,7 +1,8 @@ { - "latestVersion": "2.1.11-20-ca96ce88", - "currentHash": "f882a4d8a73512f7c658b87049bd9b85d9ef50c4241b6c17bd643e122573e732", + "latestVersion": "2.1.12-7-d0b97acd", + "currentHash": "ac40e5ff60f8e0bc7832874de529283a77f9e07d5a7d4a0e8f81e05d43e2df58", "changeList": [ + "2.1.12-7-d0b97acd, ripme now instead of ripme2 on github, ignore SSL verification option added", "2.1.11-20-ca96ce88, Commer.party next page, Imgur, E-hentai fixed, set recent language.", "2.1.10-21-c94a9543, Imagebam, Unify colons in UI, Motherless, right click menu, rgif fixed", "2.1.9-7-22e915df, HistoryMenuMouseListener right click menu, Imagefap retry logic for getFullSizedImage(), EightmusesRipper fixed", From 54ee67e103a67792df4ae6c34b2c19bbf87842bf Mon Sep 17 00:00:00 2001 From: metaprime Date: Wed, 1 Jan 2025 23:34:40 -0800 Subject: [PATCH 512/512] Update README.md: remove donation links, update maintainer list, announce merge of ripmeapp2 back into ripmeapp (#2050) Hi! I was the primary maintainer of ripmeapp/ripme from about 2016-2018. I'm pleased to see that while that repo was abandoned, a fork of the project continued over here. I'm back to poke around a bit and wanted to sync up the repo as well as make it clear who the primary developer is on the project now. I've also made an announcement post on the subreddit to point to the new active development repo. I've pushed the main of ripmeapp2/ripme to ripmeapp and I want to make this change as well to both repos to make it clear where active development is happening and who is the current primary developer. I'm not guaranteeing I'll stick around but I might poke around from time to time. I definitely don't have the time to resume any kind of primary development here. I'm also not necessarily encouraging the resurrection of ripmeapp/ripme over this fork, but if @soloturn wants to accept the maintainer invitation to that repo and merge the forks back together, I'd do what I can to support that. I have a personal (non-work) software-dev-capable computer again so working on a project like this is back on the table. This change also fixes a Markdown linter style complaint: bulleted lists are preferred to start lines with `-` to avoid ambiguity with `*` used for bold/italics. --- README.md | 98 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 58 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index f6f0868d..6334528c 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,57 @@ -# RipMe +# RipMe + [![Licensed under the MIT License](https://img.shields.io/badge/License-MIT-blue.svg)](/LICENSE.txt) [![Join the chat at https://gitter.im/RipMeApp/Lobby](https://badges.gitter.im/RipMeApp/Lobby.svg)](https://gitter.im/RipMeApp/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Subreddit](https://img.shields.io/badge/discuss-on%20reddit-blue.svg)](https://www.reddit.com/r/ripme/) ![alt Badge Status](https://github.com/ripmeapp2/ripme/actions/workflows/gradle.yml/badge.svg) -[![Coverage Status](https://coveralls.io/repos/github/RipMeApp/ripme/badge.svg?branch=master)](https://coveralls.io/github/RipMeApp/ripme?branch=master) +[![Coverage Status](https://coveralls.io/repos/github/RipMeApp/ripme/badge.svg?branch=main)](https://coveralls.io/github/RipMeApp/ripme?branch=main) -RipMe is maintained with ♥️ and in our limited free time by **[@MetaPrime](https://github.com/metaprime)**, **[@cyian-1756](https://github.com/cyian-1756)** and **[@kevin51jiang](https://github.com/kevin51jiang)**. If you'd like to contribute but aren't good with code, help keep us happy with a small contribution! Chat on [gitter](https://gitter.im/RipMeApp/Lobby). +## Recent development updates -[![Tip with PayPal](https://img.shields.io/badge/PayPal-Buy_us...-lightgrey.svg)](https://www.paypal.me/ripmeapp) -[![Tip with PayPal](https://img.shields.io/badge/coffee-%245-green.svg)](https://www.paypal.com/paypalme/ripmeapp/send?amount=5.00¤cyCode=USD&locale.x=en_US&country.x=US) -[![Tip with PayPal](https://img.shields.io/badge/beer-%2410-yellow.svg)](https://www.paypal.com/paypalme/ripmeapp/send?amount=10.00¤cyCode=USD&locale.x=en_US&country.x=US) -[![Tip with PayPal](https://img.shields.io/badge/lunch-%2420-orange.svg)](https://www.paypal.com/paypalme/ripmeapp/send?amount=20.00¤cyCode=USD&locale.x=en_US&country.x=US) -[![Tip with PayPal](https://img.shields.io/badge/dinner-%2450-red.svg)](https://www.paypal.com/paypalme/ripmeapp/send?amount=50.00¤cyCode=USD&locale.x=en_US&country.x=US) -[![Tip with PayPal](https://img.shields.io/badge/custom_amount-...-lightgrey.svg)](https://www.paypal.me/ripmeapp) +- For a while, the ripmeapp/ripme repo was inactive, but development continued at ripmeapp2/ripme. +- Now, maintainers have been updated and development has been rejoined with ripmeapp/ripme where it will continue. +- You may find a number of stale issues on ripmeapp/ripme and/or on ripmeapp2/ripme until everything is merged back together and statuses are updated. +- The current active development repo for RipMe is located at [ripmeapp/ripme](https://github.com/ripmeapp/ripme/). + +## Maintainers + +RipMe has been maintained with ♥️ and in our limited free time by the following +people, roughly in order from most recent primary developer, with current +activity marked by color of the indicator: + +- **[@soloturn](https://github.com/soloturn)** 🟢, +- **[@cyian-1756](https://github.com/cyian-1756)** 🟥, +- **[@kevin51jiang](https://github.com/kevin51jiang)** 🟥, +- **[@MetaPrime](https://github.com/metaprime)** 🟡, +- and its original creator, **[@4pr0n](https://github.com/4pr0n)** 🟥. + +If you'd like to become a maintainer, ask an active maintainer to be added to the team. + +## Contact + +Chat with the team and community on [gitter](https://gitter.im/RipMeApp/Lobby) and [reddit.com/r/ripme](https://www.reddit.com/r/ripme/) # About RipMe is an album ripper for various websites. It is a cross-platform tool that runs on your computer, and -requires Java 17. RipMe has been tested and confirmed working on Windows, Linux and MacOS. +requires Java 21 or later to run. RipMe has been tested and is confirmed working on Windows, Linux, and MacOS. ![Screenshot](https://i.imgur.com/UCQNjeg.png) ## Downloads -Download `ripme.jar` from the [latest release](https://github.com/ripmeapp2/ripme/releases). For information about running the `.jar` file, see +Download `ripme.jar` from the [latest release](https://github.com/ripmeapp2/ripme/releases). For information about running the `.jar` file, see [the How To Run wiki](https://github.com/ripmeapp/ripme/wiki/How-To-Run-RipMe). -The version number like ripme-1.7.94-17-2167aa34-feature_auto_release.jar contains a release number (1.7.94), given by -a person, the number of commits since this version (17). The commit SHA (2167aa34) uniquely references the -source code ripme was built from. If it is not built from the main branch, the branch name (feature/auto-release) is +The version number like `ripme-1.7.94-17-2167aa34-feature_auto_release.jar` contains a release number (`1.7.94`), given by +a person, the number of commits since this version (`17`). The commit SHA (`2167aa34`) uniquely references the +source code ripme was built from. If it is not built from the main branch, the branch name (`feature/auto-release`) is given. ## Installation On macOS, there is a [cask](https://github.com/Homebrew/homebrew-cask/blob/master/Casks/ripme.rb). + ``` brew install --cask ripme && xattr -d com.apple.quarantine /Applications/ripme.jar ``` @@ -44,32 +62,32 @@ brew install --cask ripme && xattr -d com.apple.quarantine /Applications/ripme.j # Features -* Quickly downloads all images in an online album. [See supported sites](https://github.com/ripmeapp/ripme/wiki/Supported-Sites) -* Easily re-rip albums to fetch new content -* Built in updater -* Skips already downloaded images by default -* Can auto skip e-hentai and nhentai albums containing certain tags. [See here for how to enable](https://github.com/RipMeApp/ripme/wiki/Config-options#nhentaiblacklisttags) -* Download a range of urls. [See here for how](https://github.com/RipMeApp/ripme/wiki/How-To-Run-RipMe#downloading-a-url-range) +- Quickly downloads all images in an online album. [See supported sites](https://github.com/ripmeapp/ripme/wiki/Supported-Sites) +- Easily re-rip albums to fetch new content +- Built in updater +- Skips already downloaded images by default +- Can auto skip e-hentai and nhentai albums containing certain tags. [See here for how to enable](https://github.com/RipMeApp/ripme/wiki/Config-options#nhentaiblacklisttags) +- Download a range of urls. [See here for how](https://github.com/RipMeApp/ripme/wiki/How-To-Run-RipMe#downloading-a-url-range) ## List of Supported Sites -* imgur -* twitter -* tumblr -* instagram -* flickr -* photobucket -* reddit -* gonewild -* motherless -* imagefap -* imagearn -* seenive -* vinebox -* 8muses -* deviantart -* xhamster -* [(more)](https://github.com/ripmeapp/ripme/wiki/Supported-Sites) +- imgur +- twitter +- tumblr +- instagram +- flickr +- photobucket +- reddit +- gonewild +- motherless +- imagefap +- imagearn +- seenive +- vinebox +- 8muses +- deviantart +- xhamster +- [(more)](https://github.com/ripmeapp/ripme/wiki/Supported-Sites) ## Not Supported? @@ -81,7 +99,7 @@ If you're a developer, you can add your own Ripper by following the wiki guide: # Compiling & Building The project uses [Gradle](https://gradle.org). To build the .jar file, -navigate to the root project directory and run at least the test you +navigate to the root project directory and run at least the test you change, e.g. Xhamster. test execution can also excluded completely: ```bash @@ -122,7 +140,7 @@ tests to break. As Java Swing will go away in future, a new GUI technology should be used. One of the candidates is [Jetpack Compose for Desktop](https://github.com/JetBrains/compose-jb/). -The library leverages the compose library for android and provides it for android, +The library leverages the compose library for android and provides it for android, desktop and web. The navigation library is not available for desktop, so Arkadii Ivanov -implemented +implemented [decompose](https://proandroiddev.com/a-comprehensive-hundred-line-navigation-for-jetpack-desktop-compose-5b723c4f256e).