From b0c9787ceadc6ce6d0d86ec794cf8b66153193c8 Mon Sep 17 00:00:00 2001 From: randomcommitter Date: Sat, 14 Sep 2019 02:51:31 +0100 Subject: [PATCH 1/4] Add erotiv.io ripper and test --- .../ripme/ripper/rippers/ErotivRipper.java | 94 +++++++++++++++++++ .../tst/ripper/rippers/ErotivRipperTest.java | 29 ++++++ 2 files changed, 123 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErotivRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java new file mode 100644 index 00000000..76c475c9 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java @@ -0,0 +1,94 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jsoup.Connection.Response; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + +/** + * + * @author randomcommitter + */ +public class ErotivRipper extends AbstractHTMLRipper { + + boolean rippingProfile; + + + public ErotivRipper (URL url) throws IOException { + super(url); + } + + @Override + public String getDomain() { + return "erotiv.io"; + } + + @Override + public String getHost() { + return "erotiv"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^https?://erotiv.io/e/([0-9]*)/?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + + throw new MalformedURLException("erotiv video not found in " + url + ", expected https://erotiv.io/e/id"); + } + + @Override + public Document getFirstPage() throws IOException { + Response resp = Http.url(this.url) + .ignoreContentType() + .response(); + + return resp.parse(); + } + + @Override + public URL sanitizeURL(URL url) throws MalformedURLException { + return new URL(url.toExternalForm().replaceAll("https?://www.erotiv.io", "https://erotiv.io")); + } + + @Override + public List getURLsFromPage(Document doc) { + List results = new ArrayList<>(); + for (Element el : doc.select("video[id=\"video-id\"] > source")) { + if (el.hasAttr("src")) { + Pattern p = Pattern.compile("/uploads/[0-9]*\\.mp4"); + Matcher m = p.matcher(el.attr("src")); + if (m.matches()) { + results.add("https://www.erotiv.io" + el.attr("src")); + } + } + + } + return results; + + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + + @Override + public boolean hasQueueSupport() { + return true; + } + +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErotivRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErotivRipperTest.java new file mode 100644 index 00000000..02319834 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErotivRipperTest.java @@ -0,0 +1,29 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.ErotivRipper; +import org.junit.jupiter.api.Test; + +public class ErotivRipperTest extends RippersTest { + @Test + public void testGetGID() throws IOException { + URL url = new URL("https://erotiv.io/e/1568314255"); + ErotivRipper ripper = new ErotivRipper(url); + assertEquals("1568314255", ripper.getGID(url)); + } + + public void testRip() throws IOException { + URL url = new URL("https://erotiv.io/e/1568314255"); + ErotivRipper ripper = new ErotivRipper(url); + testRipper(ripper); + } + @Test + public void testGetURLsFromPage() throws IOException { + URL url = new URL("https://erotiv.io/e/1568314255"); + ErotivRipper ripper = new ErotivRipper(url); + assert (1 == ripper.getURLsFromPage(ripper.getFirstPage()).size()); + } +} From 50e5e2a5f491f165983fae4fe963674dc401632e Mon Sep 17 00:00:00 2001 From: randomcommitter Date: Sat, 14 Sep 2019 03:17:44 +0100 Subject: [PATCH 2/4] Fix getGID regex to support optional "www" Also removed "www" from final video URL --- .../com/rarchives/ripme/ripper/rippers/ErotivRipper.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java index 76c475c9..cf78e65d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java @@ -24,7 +24,6 @@ public class ErotivRipper extends AbstractHTMLRipper { boolean rippingProfile; - public ErotivRipper (URL url) throws IOException { super(url); } @@ -41,7 +40,7 @@ public class ErotivRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://erotiv.io/e/([0-9]*)/?$"); + Pattern p = Pattern.compile("^https?://(?:www.)?erotiv.io/e/([0-9]*)/?$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); @@ -72,7 +71,7 @@ public class ErotivRipper extends AbstractHTMLRipper { Pattern p = Pattern.compile("/uploads/[0-9]*\\.mp4"); Matcher m = p.matcher(el.attr("src")); if (m.matches()) { - results.add("https://www.erotiv.io" + el.attr("src")); + results.add("https://erotiv.io" + el.attr("src")); } } From d64162c636a43c864ffb44156bf616b4b7879490 Mon Sep 17 00:00:00 2001 From: randomcommitter Date: Sun, 15 Sep 2019 23:11:11 +0100 Subject: [PATCH 3/4] Fix indentation --- .../ripme/ripper/rippers/ErotivRipper.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java index cf78e65d..10e73346 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ErotivRipper.java @@ -24,18 +24,18 @@ public class ErotivRipper extends AbstractHTMLRipper { boolean rippingProfile; - public ErotivRipper (URL url) throws IOException { + public ErotivRipper(URL url) throws IOException { super(url); } @Override public String getDomain() { - return "erotiv.io"; + return "erotiv.io"; } @Override public String getHost() { - return "erotiv"; + return "erotiv"; } @Override @@ -51,9 +51,7 @@ public class ErotivRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - Response resp = Http.url(this.url) - .ignoreContentType() - .response(); + Response resp = Http.url(this.url).ignoreContentType().response(); return resp.parse(); } @@ -67,15 +65,15 @@ public class ErotivRipper extends AbstractHTMLRipper { public List getURLsFromPage(Document doc) { List results = new ArrayList<>(); for (Element el : doc.select("video[id=\"video-id\"] > source")) { - if (el.hasAttr("src")) { + if (el.hasAttr("src")) { Pattern p = Pattern.compile("/uploads/[0-9]*\\.mp4"); Matcher m = p.matcher(el.attr("src")); if (m.matches()) { results.add("https://erotiv.io" + el.attr("src")); } - } + } - } + } return results; } From 15a57a6e9b86626a4bd79b15be52e0edd65a3930 Mon Sep 17 00:00:00 2001 From: randomcommitter Date: Sun, 15 Sep 2019 23:11:45 +0100 Subject: [PATCH 4/4] Fix indentation, remove unused import, avoid deprecated method --- .../tst/ripper/rippers/ErotivRipperTest.java | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErotivRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErotivRipperTest.java index 02319834..432b7fb9 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErotivRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErotivRipperTest.java @@ -1,29 +1,29 @@ package com.rarchives.ripme.tst.ripper.rippers; import java.io.IOException; -import java.net.MalformedURLException; import java.net.URL; import com.rarchives.ripme.ripper.rippers.ErotivRipper; import org.junit.jupiter.api.Test; public class ErotivRipperTest extends RippersTest { - @Test - public void testGetGID() throws IOException { - URL url = new URL("https://erotiv.io/e/1568314255"); - ErotivRipper ripper = new ErotivRipper(url); - assertEquals("1568314255", ripper.getGID(url)); - } + @Test + public void testGetGID() throws IOException { + URL url = new URL("https://erotiv.io/e/1568314255"); + ErotivRipper ripper = new ErotivRipper(url); + assert("1568314255".equals(ripper.getGID(url))); + } - public void testRip() throws IOException { - URL url = new URL("https://erotiv.io/e/1568314255"); - ErotivRipper ripper = new ErotivRipper(url); - testRipper(ripper); - } - @Test - public void testGetURLsFromPage() throws IOException { - URL url = new URL("https://erotiv.io/e/1568314255"); - ErotivRipper ripper = new ErotivRipper(url); - assert (1 == ripper.getURLsFromPage(ripper.getFirstPage()).size()); - } + public void testRip() throws IOException { + URL url = new URL("https://erotiv.io/e/1568314255"); + ErotivRipper ripper = new ErotivRipper(url); + testRipper(ripper); + } + + @Test + public void testGetURLsFromPage() throws IOException { + URL url = new URL("https://erotiv.io/e/1568314255"); + ErotivRipper ripper = new ErotivRipper(url); + assert(1 == ripper.getURLsFromPage(ripper.getFirstPage()).size()); + } }