Merge pull request #1870 from ripmeapp2/for-upstream

For upstream

.github/workflows/maven.yml

@@ -9,7 +9,11 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macOS-latest]
-        java: [1.8, 1.14]
+        java: [1.8]
+        include: # test newest java on one os only
+          - os: ubuntu-latest
+            java: 1.15
+
 
     steps:
     - uses: actions/checkout@v1

build.gradle.kts

@@ -35,6 +35,7 @@ java {
 }
 
 tasks.withType<Jar> {
+  duplicatesStrategy = DuplicatesStrategy.INCLUDE
   manifest {
     attributes["Main-Class"] = "com.rarchives.ripme.App"
   }

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.2-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.0-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

pom.xml

@@ -132,7 +132,7 @@
         <!-- At time of writing: JaCoCo is (allegedly) the only coverage report generator that supports Java 8 -->
         <groupId>org.jacoco</groupId>
         <artifactId>jacoco-maven-plugin</artifactId>
-        <version>0.8.5</version>
+        <version>0.8.6</version>
         <executions>
           <execution>
             <id>prepare-agent</id>

src/main/java/com/rarchives/ripme/ripper/rippers/DrawcrowdRipper.java

@@ -1,91 +0,0 @@
-package com.rarchives.ripme.ripper.rippers;
-
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
-
-import com.rarchives.ripme.ripper.AbstractHTMLRipper;
-import com.rarchives.ripme.utils.Http;
-
-public class DrawcrowdRipper extends AbstractHTMLRipper {
-
-    public DrawcrowdRipper(URL url) throws IOException {
-        super(url);
-    }
-
-    @Override
-    public String getHost() {
-        return "drawcrowd";
-    }
-    @Override
-    public String getDomain() {
-        return "drawcrowd.com";
-    }
-
-    @Override
-    public String getGID(URL url) throws MalformedURLException {
-        Pattern p; Matcher m;
-
-        p = Pattern.compile("^.*drawcrowd.com/projects/.*$");
-        m = p.matcher(url.toExternalForm());
-        if (m.matches()) {
-            throw new MalformedURLException("Cannot rip drawcrowd.com/projects/ pages");
-        }
-
-        p = Pattern.compile("^.*drawcrowd.com/([a-zA-Z0-9\\-_]+).*$");
-        m = p.matcher(url.toExternalForm());
-        if (m.matches()) {
-            return m.group(1);
-        }
-
-        throw new MalformedURLException(
-                "Expected drawcrowd.com gallery format: "
-                        + "drawcrowd.com/username"
-                        + " Got: " + url);
-    }
-
-    @Override
-    public Document getFirstPage() throws IOException {
-        return Http.url(this.url).get();
-    }
-
-    @Override
-    public Document getNextPage(Document doc) throws IOException {
-        Elements loadMore = doc.select("a#load-more");
-        if (loadMore.isEmpty()) {
-            throw new IOException("No next page found");
-        }
-        if (!sleep(1000)) {
-            throw new IOException("Interrupted while waiting for next page");
-        }
-        String nextPage = "http://drawcrowd.com" + loadMore.get(0).attr("href");
-        return Http.url(nextPage).get();
-    }
-
-    @Override
-    public List<String> getURLsFromPage(Document page) {
-        List<String> imageURLs = new ArrayList<>();
-        for (Element thumb : page.select("div.item.asset img")) {
-            String image = thumb.attr("src");
-            image = image
-                    .replaceAll("/medium/", "/large/")
-                    .replaceAll("/small/", "/large/");
-            imageURLs.add(image);
-        }
-        return imageURLs;
-    }
-
-    @Override
-    public void downloadURL(URL url, int index) {
-        addURLToDownload(url, getPrefix(index));
-    }
-
-}

src/main/java/com/rarchives/ripme/ripper/rippers/PorncomixDotOneRipper.java

@@ -1,68 +0,0 @@
-package com.rarchives.ripme.ripper.rippers;
-
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
-
-import com.rarchives.ripme.ripper.AbstractHTMLRipper;
-import com.rarchives.ripme.utils.Http;
-
-public class PorncomixDotOneRipper extends AbstractHTMLRipper {
-
-    public PorncomixDotOneRipper(URL url) throws IOException {
-        super(url);
-    }
-
-    @Override
-    public String getHost() {
-        return "porncomix";
-    }
-
-    @Override
-    public String getDomain() {
-        return "porncomix.one";
-    }
-
-    @Override
-    public String getGID(URL url) throws MalformedURLException {
-        Pattern p = Pattern.compile("https?://www.porncomix.one/gallery/([a-zA-Z0-9_\\-]*)/?$");
-        Matcher m = p.matcher(url.toExternalForm());
-        if (m.matches()) {
-            return m.group(1);
-        }
-        throw new MalformedURLException("Expected proncomix URL format: " +
-                "porncomix.one/gallery/comic - got " + url + " instead");
-    }
-
-    @Override
-    public Document getFirstPage() throws IOException {
-        // "url" is an instance field of the superclass
-        return Http.url(url).get();
-    }
-
-    @Override
-    public List<String> getURLsFromPage(Document doc) {
-        List<String> result = new ArrayList<>();
-        // We have 2 loops here to cover all the different album types
-        for (Element el : doc.select(".dgwt-jg-item > a")) {
-            result.add(el.attr("href"));
-        }
-        for (Element el : doc.select(".unite-gallery > img")) {
-            result.add(el.attr("data-image"));
-
-        }
-        return result;
-    }
-
-    @Override
-    public void downloadURL(URL url, int index) {
-        addURLToDownload(url, getPrefix(index));
-    }
-}

src/main/java/com/rarchives/ripme/ui/MainWindow.java

@@ -1183,6 +1183,7 @@ public final class MainWindow implements Runnable, RipStatusHandler {
                 // Guess rip history based on rip folder
                 String[] dirs = Utils.getWorkingDirectory()
                         .list((dir, file) -> new File(dir.getAbsolutePath() + File.separator + file).isDirectory());
+                if (dirs != null) {
                     for (String dir : dirs) {
                         String url = RipUtils.urlFromDirectoryName(dir);
                         if (url != null) {
@@ -1195,6 +1196,7 @@ public final class MainWindow implements Runnable, RipStatusHandler {
                 }
             }
         }
+    }
 
     private void saveHistory() {
         Path historyFile = Paths.get(Utils.getConfigDir() + File.separator + "history.json");

src/test/java/com/rarchives/ripme/tst/ripper/rippers/AerisdiesRipperTest.java

@@ -6,16 +6,19 @@ import java.net.URL;
 import com.rarchives.ripme.ripper.rippers.AerisdiesRipper;
 
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 public class AerisdiesRipperTest extends RippersTest {
     @Test
+    @Tag("flaky")
     public void testAlbum() throws IOException {
         AerisdiesRipper ripper = new AerisdiesRipper(new URL("http://www.aerisdies.com/html/lb/alb_1097_1.html"));
         testRipper(ripper);
     }
 
     @Test
+    @Tag("flaky")
     public void testSubAlbum() throws IOException {
         AerisdiesRipper ripper = new AerisdiesRipper(new URL("http://www.aerisdies.com/html/lb/alb_3692_1.html"));
         testRipper(ripper);

src/test/java/com/rarchives/ripme/tst/ripper/rippers/ArtStationRipperTest.java

@@ -7,12 +7,13 @@ import java.util.List;
 
 import com.rarchives.ripme.ripper.rippers.ArtStationRipper;
 
-import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 public class ArtStationRipperTest extends RippersTest {
 
     @Test
+    @Tag("flaky")
     public void testArtStationProjects() throws IOException {
         List<URL> contentURLs = new ArrayList<>();
         contentURLs.add(new URL("https://www.artstation.com/artwork/the-dwarf-mortar"));
@@ -25,7 +26,7 @@ public class ArtStationRipperTest extends RippersTest {
     }
 
     @Test
-    @Disabled("Failed with cloudflare protection")
+    @Tag("flaky")
     public void testArtStationUserProfiles() throws IOException {
         List<URL> contentURLs = new ArrayList<>();
         contentURLs.add(new URL("https://www.artstation.com/heitoramatsu"));

src/test/java/com/rarchives/ripme/tst/ripper/rippers/DrawcrowdRipperTest.java

@@ -1,19 +0,0 @@
-package com.rarchives.ripme.tst.ripper.rippers;
-
-import java.io.IOException;
-import java.net.URL;
-
-import com.rarchives.ripme.ripper.rippers.DrawcrowdRipper;
-
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-public class DrawcrowdRipperTest extends RippersTest {
-    @Test
-    @Disabled("https://github.com/RipMeApp/ripme/issues/304 -- Drawcrowd broken (site changed)")
-    public void testRip() throws IOException {
-        DrawcrowdRipper ripper = new DrawcrowdRipper(new URL("https://drawcrowd.com/rabbiteyes"));
-        testRipper(ripper);
-    }
-
-}

src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java

@@ -4,10 +4,12 @@ import java.io.IOException;
 import java.net.URL;
 
 import com.rarchives.ripme.ripper.rippers.Hentai2readRipper;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 public class Hentai2readRipperTest extends RippersTest {
     @Test
+    @Tag("flaky")
     public void testHentai2readAlbum() throws IOException {
         Hentai2readRipper ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/1/"));
         testRipper(ripper);

src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentaicafeRipperTest.java

@@ -4,16 +4,19 @@ import java.io.IOException;
 import java.net.URL;
 
 import com.rarchives.ripme.ripper.rippers.HentaiCafeRipper;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 public class HentaicafeRipperTest extends RippersTest {
     @Test
+    @Tag("flaky")
     public void testHentaiCafeAlbum() throws IOException {
         HentaiCafeRipper ripper = new HentaiCafeRipper(new URL("https://hentai.cafe/kikuta-the-oni-in-the-room/"));
         testRipper(ripper);
     }
     // This album has a line break (<br />) in the url. Test it to make sure ripme can handle these invalid urls
     @Test
+    @Tag("flaky")
     public void testAlbumWithInvalidChars() throws IOException {
         HentaiCafeRipper ripper = new HentaiCafeRipper(new URL("https://hentai.cafe/chobipero-club/"));
         testRipper(ripper);

src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java

@@ -8,10 +8,12 @@ import java.util.List;
 import com.rarchives.ripme.ripper.rippers.HentaiNexusRipper;
 import org.json.JSONObject;
 import org.junit.Assert;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 public class HentainexusRipperTest extends RippersTest {
     @Test
+    @Tag("flaky")
     public void testHentaiNexusJson() throws IOException {
         List<URL> testURLs = new ArrayList<>();
         testURLs.add(new URL("https://hentainexus.com/view/9202"));

src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java

@@ -4,10 +4,12 @@ import java.io.IOException;
 import java.net.URL;
 
 import com.rarchives.ripme.ripper.rippers.ImagebamRipper;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 public class ImagebamRipperTest extends RippersTest {
     @Test
+    @Tag("flaky")
     public void testImagebamRip() throws IOException {
         ImagebamRipper ripper = new ImagebamRipper(new URL("http://www.imagebam.com/gallery/488cc796sllyf7o5srds8kpaz1t4m78i"));
         testRipper(ripper);

src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java

@@ -5,10 +5,12 @@ import java.net.URL;
 
 import com.rarchives.ripme.ripper.rippers.MotherlessRipper;
 
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 public class MotherlessRipperTest extends RippersTest {
     @Test
+    @Tag("flaky")
     public void testMotherlessAlbumRip() throws IOException {
         MotherlessRipper ripper = new MotherlessRipper(new URL("https://motherless.com/G1168D90"));
         testRipper(ripper);

src/test/java/com/rarchives/ripme/tst/ripper/rippers/NhentaiRipperTest.java

@@ -7,6 +7,7 @@ import java.util.List;
 import com.rarchives.ripme.ripper.rippers.NhentaiRipper;
 import com.rarchives.ripme.utils.RipUtils;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 public class NhentaiRipperTest extends RippersTest {
@@ -22,6 +23,7 @@ public class NhentaiRipperTest extends RippersTest {
 
     // Test the tag black listing
     @Test
+    @Tag("flaky")
     public void testTagBlackList()  throws IOException {
         URL url = new URL("https://nhentai.net/g/233295/");
         NhentaiRipper ripper = new NhentaiRipper(url);

src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixDotOneRipperTest.java

@@ -1,15 +0,0 @@
-package com.rarchives.ripme.tst.ripper.rippers;
-
-import java.io.IOException;
-import java.net.URL;
-
-import com.rarchives.ripme.ripper.rippers.PorncomixDotOneRipper;
-import org.junit.jupiter.api.Test;
-
-public class PorncomixDotOneRipperTest extends RippersTest {
-    @Test
-    public void testPorncomixAlbum() throws IOException {
-        PorncomixDotOneRipper ripper = new PorncomixDotOneRipper(new URL("https://www.porncomix.one/gallery/blacknwhite-make-america-great-again"));
-        testRipper(ripper);
-    }
-}

src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java

@@ -8,6 +8,7 @@ import com.rarchives.ripme.utils.Http;
 import com.rarchives.ripme.utils.Utils;
 import org.jsoup.nodes.Document;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 public class PornhubRipperTest extends RippersTest {
@@ -28,6 +29,7 @@ public class PornhubRipperTest extends RippersTest {
     }
 
     @Test
+    @Tag("flaky")
     public void testGetNextPage() throws IOException {
         String baseURL = "https://www.pornhub.com/album/30687901";
         PornhubRipper ripper = new PornhubRipper(new URL(baseURL));

src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java

@@ -36,6 +36,7 @@ public class RedgifsRipperTest extends RippersTest {
      * @throws IOException
      */
     @Test
+    @Tag("flaky")
     public void testRedgifsProfile() throws IOException {
         RedgifsRipper ripper  = new RedgifsRipper(new URL("https://redgifs.com/users/margo_monty"));
         testRipper(ripper);

src/test/java/com/rarchives/ripme/tst/ripper/rippers/SmuttyRipperTest.java

@@ -5,10 +5,12 @@ import java.net.URL;
 
 import com.rarchives.ripme.ripper.rippers.SmuttyRipper;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 public class SmuttyRipperTest extends RippersTest {
     @Test
+    @Tag("flaky")
     public void testRip() throws IOException {
         SmuttyRipper ripper = new SmuttyRipper(new URL("https://smutty.com/user/QUIGON/"));
         testRipper(ripper);

src/test/java/com/rarchives/ripme/tst/ripper/rippers/ZizkiRipperTest.java

@@ -5,20 +5,27 @@ import java.net.URL;
 
 import com.rarchives.ripme.ripper.rippers.ZizkiRipper;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 public class ZizkiRipperTest extends RippersTest {
+
+    @Test
+    @Tag("flaky")
     public void testRip() throws IOException {
         ZizkiRipper ripper = new ZizkiRipper(new URL("http://zizki.com/dee-chorde/we-got-spirit"));
         testRipper(ripper);
     }
 
+    @Test
     public void testGetGID() throws IOException {
         URL url = new URL("http://zizki.com/dee-chorde/we-got-spirit");
         ZizkiRipper ripper = new ZizkiRipper(url);
         Assertions.assertEquals("dee-chorde", ripper.getGID(url));
     }
+
     @Test
+    @Tag("flaky")
     public void testAlbumTitle() throws IOException {
         URL url = new URL("http://zizki.com/dee-chorde/we-got-spirit");
         ZizkiRipper ripper = new ZizkiRipper(url);