From 0e6a6e3cebe6680a507c020d99568d51ed349f3e Mon Sep 17 00:00:00 2001 From: Tushar Date: Mon, 21 Jan 2019 09:42:11 +0530 Subject: [PATCH] Fixed indentation. --- .../ripper/rippers/HentaidudeRipper.java | 260 +++++++++--------- .../ripper/rippers/MyhentaigalleryRipper.java | 82 +++--- .../ripper/rippers/HqpornerRipperTest.java | 102 +++---- .../tst/ripper/rippers/ModelxRipperTest.java | 22 +- .../rippers/MyhentaigalleryRipperTest.java | 20 +- 5 files changed, 242 insertions(+), 244 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java index bb2c6dbd..cf3656c1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java @@ -20,152 +20,150 @@ import java.util.regex.Pattern; public class HentaidudeRipper extends AbstractSingleFileRipper { - private Pattern p1 = Pattern.compile("https?://hentaidude\\.com/([a-zA-Z0-9_-]*)/?$"); // to match URLs. - private Pattern p2 = Pattern.compile("data:\\s?(\\{.*?\\})", Pattern.DOTALL); + private Pattern p1 = Pattern.compile("https?://hentaidude\\.com/([a-zA-Z0-9_-]*)/?$"); // to match URLs. + private Pattern p2 = Pattern.compile("data:\\s?(\\{.*?\\})", Pattern.DOTALL); - public DownloadThreadPool hentaidudeThreadPool = new DownloadThreadPool("hentaidudeThreadPool"); + public DownloadThreadPool hentaidudeThreadPool = new DownloadThreadPool("hentaidudeThreadPool"); - public HentaidudeRipper(URL url) throws IOException { - super(url); + public HentaidudeRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "hentaidude"; + } + + @Override + public String getDomain() { + return "hentaidude.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + + Matcher m = p1.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException( + "Expected hqporner URL format: " + "hentaidude.com/VIDEO - got " + url + " instead"); + } + + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } + + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + Matcher m1 = p1.matcher(url.toString()); + if (m1.matches()) { + result.add(url.toString()); + } + + // Can add support for search page. + return result; + } + + @Override + public boolean tryResumeDownload() { + return true; + } + + @Override + public void downloadURL(URL url, int index) { + // addURLToDownload(url, "", "", "", null, getVideoName(), "mp4"); + hentaidudeThreadPool.addThread(new HentaidudeDownloadThread(url, index)); + } + + @Override + public DownloadThreadPool getThreadPool() { + return hentaidudeThreadPool; + } + + private class HentaidudeDownloadThread extends Thread { + + private URL url; + + public HentaidudeDownloadThread(URL url, int index) { + this.url = url; + // this.index = index; } @Override - public String getHost() { - return "hentaidude"; + public void run() { + try { + Document doc = Http.url(url).get(); + URL videoSourceUrl = new URL(getVideoUrl(doc)); + addURLToDownload(videoSourceUrl, "", "", "", null, getVideoName(), "mp4"); + } catch (Exception e) { + LOGGER.error("Could not get video url for " + getVideoName(), e); + } } - @Override - public String getDomain() { - return "hentaidude.com"; + private String getVideoName() { + try { + return getGID(url); + } catch (MalformedURLException e) { + LOGGER.error("Unable to get video title from " + url.toExternalForm()); + e.printStackTrace(); + } + return "unknown"; } - @Override - public String getGID(URL url) throws MalformedURLException { + /* + * TO find data object: $.ajax({ url: + * 'https://hentaidude.com/wp-admin/admin-ajax.php', type: 'post', data: { + * action: 'msv-get-sources', id: '48227', nonce: '907f1bd45c' } + */ + public String getVideoUrl(Document doc) throws IOException { + String jsonString = null; + Matcher m = p2.matcher(doc.html()); - Matcher m = p1.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); + while (m.find()) { + jsonString = m.group(1); + if (jsonString.contains("msv-get-sources")) + break; + } + + if (jsonString != null) { + // send POST request to https://hentaidude.com/wp-admin/admin-ajax.php with the + // data object parameters. + JSONObject dataObject = new JSONObject(jsonString); + Map dataMap = new HashMap<>(); + for (String key : JSONObject.getNames(dataObject)) { + dataMap.put(key, dataObject.getString(key)); } - throw new MalformedURLException( - "Expected hqporner URL format: " + "hentaidude.com/VIDEO - got " + url + " instead"); - } + JSONObject jsonResopnse = Http.url("https://hentaidude.com/wp-admin/admin-ajax.php").data(dataMap) + .method(Method.POST).getJSON(); + // return source url from below JSON. + /* + * success true sources { video-source-0 + * https://cdn1.hentaidude.com/index.php?data= + * 2f4a576957694872754d6736466f6c585579704b4d584e4a434372546c51346d4f4c697a6c734f6678307a59324c5458624f4675664863323768397a3371452f41384b62375246643243466f744447536b2b6250565a3859306a41506d366942713066336c6659386d78513d + * video-source-1 } + */ - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } - - @Override - public List getURLsFromPage(Document doc) { - List result = new ArrayList<>(); - Matcher m1 = p1.matcher(url.toString()); - if (m1.matches()) { - result.add(url.toString()); - } - - //Can add support for search page. - return result; - } - - @Override - public boolean tryResumeDownload() { - return true; - } - - @Override - public void downloadURL(URL url, int index) { - // addURLToDownload(url, "", "", "", null, getVideoName(), "mp4"); - hentaidudeThreadPool.addThread(new HentaidudeDownloadThread(url, index)); - } - - @Override - public DownloadThreadPool getThreadPool() { - return hentaidudeThreadPool; - } - - private class HentaidudeDownloadThread extends Thread { - - private URL url; - - public HentaidudeDownloadThread(URL url, int index) { - this.url = url; - //this.index = index; - } - - @Override - public void run() { - try { - Document doc = Http.url(url).get(); - URL videoSourceUrl = new URL(getVideoUrl(doc)); - addURLToDownload(videoSourceUrl, "", "", "", null, getVideoName(), "mp4"); - } catch (Exception e) { - LOGGER.error("Could not get video url for " + getVideoName(), e); + if (jsonResopnse.getBoolean("success")) { + // get the hentaidude video source + for (String key : JSONObject.getNames(jsonResopnse.getJSONObject("sources"))) { + if (jsonResopnse.getJSONObject("sources").getString(key).contains("hentaidude.com")) { + return jsonResopnse.getJSONObject("sources").getString(key); } + } + } else { + throw new IOException("Could not get video url from JSON response."); } - private String getVideoName() { - try { - return getGID(url); - } catch (MalformedURLException e) { - LOGGER.error("Unable to get video title from " + url.toExternalForm()); - e.printStackTrace(); - } - return "unknown"; - } + } - /* TO find data object: - $.ajax({ - url: 'https://hentaidude.com/wp-admin/admin-ajax.php', - type: 'post', - data: { - action: 'msv-get-sources', - id: '48227', - nonce: '907f1bd45c' - } - */ - public String getVideoUrl(Document doc) throws IOException { - String jsonString = null; - Matcher m = p2.matcher(doc.html()); - - while (m.find()) { - jsonString = m.group(1); - if (jsonString.contains("msv-get-sources")) - break; - } - - if (jsonString != null) { - // send POST request to https://hentaidude.com/wp-admin/admin-ajax.php with the data object parameters. - JSONObject dataObject = new JSONObject(jsonString); - Map dataMap = new HashMap<>(); - for (String key : JSONObject.getNames(dataObject)) { - dataMap.put(key, dataObject.getString(key)); - } - JSONObject jsonResopnse = Http.url("https://hentaidude.com/wp-admin/admin-ajax.php").data(dataMap) - .method(Method.POST).getJSON(); - // return source url from below JSON. - /* - success true - sources { - video-source-0 https://cdn1.hentaidude.com/index.php?data=2f4a576957694872754d6736466f6c585579704b4d584e4a434372546c51346d4f4c697a6c734f6678307a59324c5458624f4675664863323768397a3371452f41384b62375246643243466f744447536b2b6250565a3859306a41506d366942713066336c6659386d78513d - video-source-1 - } - */ - - if (jsonResopnse.getBoolean("success")) { - // get the hentaidude video source - for (String key : JSONObject.getNames(jsonResopnse.getJSONObject("sources"))) { - if (jsonResopnse.getJSONObject("sources").getString(key).contains("hentaidude.com")) { - return jsonResopnse.getJSONObject("sources").getString(key); - } - } - } else { - throw new IOException("Could not get video url from JSON response."); - } - - } - - throw new IOException("Could not get video download url."); - } + throw new IOException("Could not get video download url."); } + } } \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaigalleryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaigalleryRipper.java index c2bf48d4..d8422942 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaigalleryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MyhentaigalleryRipper.java @@ -14,53 +14,53 @@ import org.jsoup.nodes.Element; public class MyhentaigalleryRipper extends AbstractHTMLRipper { - public MyhentaigalleryRipper(URL url) throws IOException { - super(url); - } + public MyhentaigalleryRipper(URL url) throws IOException { + super(url); + } - @Override - public String getHost() { - return "myhentaigallery"; - } + @Override + public String getHost() { + return "myhentaigallery"; + } - @Override - public String getDomain() { - return "myhentaigallery.com"; - } + @Override + public String getDomain() { + return "myhentaigallery.com"; + } - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("https://myhentaigallery.com/gallery/thumbnails/([0-9]+)/?$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("https://myhentaigallery.com/gallery/thumbnails/([0-9]+)/?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } - throw new MalformedURLException("Expected myhentaicomics.com URL format: " - + "myhentaigallery.com/gallery/thumbnails/ID - got " + url + " instead"); - } + throw new MalformedURLException("Expected myhentaicomics.com URL format: " + + "myhentaigallery.com/gallery/thumbnails/ID - got " + url + " instead"); + } - @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - return Http.url(url).get(); - } + @Override + public Document getFirstPage() throws IOException { + // "url" is an instance field of the superclass + return Http.url(url).get(); + } - @Override - public List getURLsFromPage(Document doc) { - List result = new ArrayList<>(); - for (Element el : doc.select(".comic-thumb > img")) { - String imageSource = el.attr("src"); - // We replace thumbs with resizes so we can the full sized images - imageSource = imageSource.replace("thumbnail", "original"); - result.add(imageSource); - } - return result; - } + @Override + public List getURLsFromPage(Document doc) { + List result = new ArrayList<>(); + for (Element el : doc.select(".comic-thumb > img")) { + String imageSource = el.attr("src"); + // We replace thumbs with resizes so we can the full sized images + imageSource = imageSource.replace("thumbnail", "original"); + result.add(imageSource); + } + return result; + } - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HqpornerRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HqpornerRipperTest.java index 92efddff..e1b730e1 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HqpornerRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HqpornerRipperTest.java @@ -8,62 +8,62 @@ import java.net.URL; public class HqpornerRipperTest extends RippersTest { - public void testRip() throws IOException { - if (Utils.getConfigBoolean("test.run_flaky_tests", false)) { - HqpornerRipper ripper = new HqpornerRipper( - new URL("https://hqporner.com/hdporn/84636-pool_lesson_with_a_cheating_husband.html")); - testRipper(ripper); - } - } + public void testRip() throws IOException { + if (Utils.getConfigBoolean("test.run_flaky_tests", false)) { + HqpornerRipper ripper = new HqpornerRipper( + new URL("https://hqporner.com/hdporn/84636-pool_lesson_with_a_cheating_husband.html")); + testRipper(ripper); + } + } - public void testGetGID() throws IOException { - URL poolURL = new URL("https://hqporner.com/hdporn/84636-pool_lesson_with_a_cheating_husband.html"); - HqpornerRipper ripper = new HqpornerRipper(poolURL); - assertEquals("84636-pool_lesson_with_a_cheating_husband", ripper.getGID(poolURL)); - } + public void testGetGID() throws IOException { + URL poolURL = new URL("https://hqporner.com/hdporn/84636-pool_lesson_with_a_cheating_husband.html"); + HqpornerRipper ripper = new HqpornerRipper(poolURL); + assertEquals("84636-pool_lesson_with_a_cheating_husband", ripper.getGID(poolURL)); + } - public void testGetURLsFromPage() throws IOException { - URL actressUrl = new URL("https://hqporner.com/actress/kali-roses"); - HqpornerRipper ripper = new HqpornerRipper(actressUrl); - assert (ripper.getURLsFromPage(ripper.getFirstPage()).size() >= 2); - } + public void testGetURLsFromPage() throws IOException { + URL actressUrl = new URL("https://hqporner.com/actress/kali-roses"); + HqpornerRipper ripper = new HqpornerRipper(actressUrl); + assert (ripper.getURLsFromPage(ripper.getFirstPage()).size() >= 2); + } - public void testGetNextPage() throws IOException { - URL multiPageUrl = new URL("https://hqporner.com/category/tattooed"); - HqpornerRipper multiPageRipper = new HqpornerRipper(multiPageUrl); - assert (multiPageRipper.getNextPage(multiPageRipper.getFirstPage()) != null); + public void testGetNextPage() throws IOException { + URL multiPageUrl = new URL("https://hqporner.com/category/tattooed"); + HqpornerRipper multiPageRipper = new HqpornerRipper(multiPageUrl); + assert (multiPageRipper.getNextPage(multiPageRipper.getFirstPage()) != null); - URL singlePageUrl = new URL("https://hqporner.com/actress/amy-reid"); - HqpornerRipper ripper = new HqpornerRipper(singlePageUrl); - try { - ripper.getNextPage(ripper.getFirstPage()); - } catch (IOException e) { - assertEquals(e.getMessage(), "No next page found."); - } - } + URL singlePageUrl = new URL("https://hqporner.com/actress/amy-reid"); + HqpornerRipper ripper = new HqpornerRipper(singlePageUrl); + try { + ripper.getNextPage(ripper.getFirstPage()); + } catch (IOException e) { + assertEquals(e.getMessage(), "No next page found."); + } + } - public void testMyDaddyVideoHost() throws IOException { - if (Utils.getConfigBoolean("test.run_flaky_tests", false)) { - URL myDaddyUrl = new URL("https://hqporner.com/hdporn/84636-pool_lesson_with_a_cheating_husband.html"); - HqpornerRipper myDaddyRipper = new HqpornerRipper(myDaddyUrl); - testRipper(myDaddyRipper); - } - } + public void testMyDaddyVideoHost() throws IOException { + if (Utils.getConfigBoolean("test.run_flaky_tests", false)) { + URL myDaddyUrl = new URL("https://hqporner.com/hdporn/84636-pool_lesson_with_a_cheating_husband.html"); + HqpornerRipper myDaddyRipper = new HqpornerRipper(myDaddyUrl); + testRipper(myDaddyRipper); + } + } - public void testFlyFlvVideoHost() throws IOException { - if (Utils.getConfigBoolean("test.run_flaky_tests", false)) { - URL flyFlvUrl = new URL( - "https://hqporner.com/hdporn/69862-bangbros_-_amy_reid_taking_off_a_tight_sexy_swimsuit.html"); - HqpornerRipper flyFlvRipper = new HqpornerRipper(flyFlvUrl); - testRipper(flyFlvRipper); - } - } + public void testFlyFlvVideoHost() throws IOException { + if (Utils.getConfigBoolean("test.run_flaky_tests", false)) { + URL flyFlvUrl = new URL( + "https://hqporner.com/hdporn/69862-bangbros_-_amy_reid_taking_off_a_tight_sexy_swimsuit.html"); + HqpornerRipper flyFlvRipper = new HqpornerRipper(flyFlvUrl); + testRipper(flyFlvRipper); + } + } - public void testUnknownVideoHost() throws IOException { - if (Utils.getConfigBoolean("test.run_flaky_tests", false)) { - URL unknownHostUrl = new URL("https://hqporner.com/hdporn/79528-Kayden_Kross_-_Serious_Masturbation.html"); // howq.cc - HqpornerRipper unknownHostRipper = new HqpornerRipper(unknownHostUrl); - testRipper(unknownHostRipper); - } - } + public void testUnknownVideoHost() throws IOException { + if (Utils.getConfigBoolean("test.run_flaky_tests", false)) { + URL unknownHostUrl = new URL("https://hqporner.com/hdporn/79528-Kayden_Kross_-_Serious_Masturbation.html"); // howq.cc + HqpornerRipper unknownHostRipper = new HqpornerRipper(unknownHostUrl); + testRipper(unknownHostRipper); + } + } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelxRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelxRipperTest.java index 046c7165..8abc0a7c 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelxRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ModelxRipperTest.java @@ -6,15 +6,15 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.ModelxRipper; public class ModelxRipperTest extends RippersTest { - public void testModelxAlbum() throws IOException { - ModelxRipper ripper = new ModelxRipper(new URL( - "http://www.modelx.org/graphis-collection-2002-2016/ai-yuzuki-%e6%9f%9a%e6%9c%88%e3%81%82%e3%81%84-yuzuiro/")); - System.out.println(ripper.getGID(new URL( - "http://www.modelx.org/graphis-collection-2002-2016/ai-yuzuki-%e6%9f%9a%e6%9c%88%e3%81%82%e3%81%84-yuzuiro/"))); - /* - * ModelxRipper domain has been changes. - * Commenting to avoid build failure. - */ - // testRipper(ripper); - } + public void testModelxAlbum() throws IOException { + ModelxRipper ripper = new ModelxRipper(new URL( + "http://www.modelx.org/graphis-collection-2002-2016/ai-yuzuki-%e6%9f%9a%e6%9c%88%e3%81%82%e3%81%84-yuzuiro/")); + System.out.println(ripper.getGID(new URL( + "http://www.modelx.org/graphis-collection-2002-2016/ai-yuzuki-%e6%9f%9a%e6%9c%88%e3%81%82%e3%81%84-yuzuiro/"))); + /* + * ModelxRipper domain has been changes. + * Commenting to avoid build failure. + */ + // testRipper(ripper); + } } \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaigalleryRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaigalleryRipperTest.java index ebceef58..fc150c7e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaigalleryRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MyhentaigalleryRipperTest.java @@ -7,15 +7,15 @@ import com.rarchives.ripme.ripper.rippers.MyhentaigalleryRipper; public class MyhentaigalleryRipperTest extends RippersTest { - public void testMyhentaigalleryAlbum() throws IOException { - MyhentaigalleryRipper ripper = new MyhentaigalleryRipper( - new URL("https://myhentaigallery.com/gallery/thumbnails/9201")); - testRipper(ripper); - } + public void testMyhentaigalleryAlbum() throws IOException { + MyhentaigalleryRipper ripper = new MyhentaigalleryRipper( + new URL("https://myhentaigallery.com/gallery/thumbnails/9201")); + testRipper(ripper); + } - public void testGetGID() throws IOException { - URL url = new URL("https://myhentaigallery.com/gallery/thumbnails/9201"); - MyhentaigalleryRipper ripper = new MyhentaigalleryRipper(url); - assertEquals("9201", ripper.getGID(url)); - } + public void testGetGID() throws IOException { + URL url = new URL("https://myhentaigallery.com/gallery/thumbnails/9201"); + MyhentaigalleryRipper ripper = new MyhentaigalleryRipper(url); + assertEquals("9201", ripper.getGID(url)); + } } \ No newline at end of file