multiple rippers java.nio

2025-08-16 10:54:09 +02:00 · 2022-01-03 16:29:19 +01:00
parent 627152853e
commit e3f6499840
5 changed files with 59 additions and 49 deletions
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java
@@ -1,10 +1,11 @@
 package com.rarchives.ripme.ripper.rippers;

-import java.io.File;
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.regex.Matcher;
@@ -330,9 +331,9 @@ public class FivehundredpxRipper extends AbstractJSONRipper {
    public void downloadURL(URL url, int index) {
        String u = url.toExternalForm();
        String[] fields = u.split("/");
-        String prefix = getPrefix(index) + fields[fields.length - 3];
-        File saveAs = new File(getWorkingDir() + File.separator + prefix + ".jpg");
-        addURLToDownload(url,  saveAs,  "", null, false);
+        String prefix = "/" + getPrefix(index) + fields[fields.length - 3];
+        Path saveAs = Paths.get(getWorkingDir() + prefix + ".jpg");
+        addURLToDownload(url,  saveAs.toFile(),  "", null, false);
    }

 }
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
@@ -1,10 +1,12 @@
 package com.rarchives.ripme.ripper.rippers;

-import java.io.File;
-import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -15,7 +17,6 @@ import java.util.regex.Pattern;
 import com.rarchives.ripme.ui.RipStatusMessage;
 import com.rarchives.ripme.utils.Utils;
 import org.jsoup.Connection.Response;
-import org.jsoup.HttpStatusException;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
@@ -91,14 +92,13 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
        String nextUrl = urlBase + nextPageUrl.first().attr("href");

        sleep(500);
-        Document nextPage = Http.url(nextUrl).cookies(cookies).get();

-        return nextPage;
+        return Http.url(nextUrl).cookies(cookies).get();
    }

    private String getImageFromPost(String url) {
        sleep(1000);
-        Document d = null;
+        Document d;
        try {
            d = Http.url(url).cookies(cookies).get();
            Elements links = d.getElementsByTag("a");
@@ -184,24 +184,22 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
        }
        String newText = "";
        String saveAs = "";
-        File saveFileAs;
+        Path saveFileAs;
        saveAs = text.split("\n")[0];
        saveAs = saveAs.replaceAll("^(\\S+)\\s+by\\s+(.*)$", "$2_$1");
        for (int i = 1;i < text.split("\n").length; i++) {
            newText = newText.replace("\\","").replace("/","").replace("~","") + "\n" + text.split("\n")[i];
        }
        try {
-            if (!subdirectory.equals("")) {
-                subdirectory = File.separator + subdirectory;
-            }
-            saveFileAs = new File(
-                    workingDir.getCanonicalPath()
+            saveFileAs = Paths.get(
+                    workingDir
+                            + "/"
                            + subdirectory
-                            + File.separator
+                            + "/"
                            + saveAs
                            + ".txt");
            // Write the file
-            FileOutputStream out = (new FileOutputStream(saveFileAs));
+            OutputStream out = Files.newOutputStream(saveFileAs);
            out.write(text.getBytes());
            out.close();
        } catch (IOException e) {
@@ -209,9 +207,13 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
            return false;
        }
        LOGGER.debug("Downloading " + url + "'s description to " + saveFileAs);
-        if (!saveFileAs.getParentFile().exists()) {
-            LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent()));
-            saveFileAs.getParentFile().mkdirs();
+        if (!Files.exists(saveFileAs.getParent())) {
+            LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent().toFile()));
+            try {
+                Files.createDirectory(saveFileAs.getParent());
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
        }
        return true;
    }
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java
@@ -3,12 +3,13 @@ package com.rarchives.ripme.ripper.rippers;
 import com.rarchives.ripme.ripper.AbstractHTMLRipper;
 import com.rarchives.ripme.utils.Http;
 import com.rarchives.ripme.utils.Utils;
-import java.io.File;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -58,7 +59,7 @@ public class PahealRipper extends AbstractHTMLRipper {
    @Override
    public Document getNextPage(Document page) throws IOException {
        for (Element e : page.select("#paginator a")) {
-            if (e.text().toLowerCase().equals("next")) {
+            if (e.text().equalsIgnoreCase("next")) {
                return Http.url(e.absUrl("href")).cookies(getCookies()).get();
            }
        }
@@ -90,12 +91,12 @@ public class PahealRipper extends AbstractHTMLRipper {
                name = name.substring(0, name.length() - ext.length());
            }

-            File outFile = new File(workingDir.getCanonicalPath()
-                + File.separator
+            Path outFile = Paths.get(workingDir
+                + "/"
                + Utils.filesystemSafe(new URI(name).getPath())
                + ext);
-            addURLToDownload(url, outFile);
-        } catch (IOException | URISyntaxException ex) {
+            addURLToDownload(url, outFile.toFile());
+        } catch (URISyntaxException ex) {
            logger.error("Error while downloading URL " + url, ex);
        }
    }
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java
@@ -1,9 +1,9 @@
 package com.rarchives.ripme.ripper.rippers;

-import java.io.File;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.regex.Matcher;
@@ -74,7 +74,7 @@ public class PornhubRipper extends AbstractHTMLRipper {

    @Override
    protected void downloadURL(URL url, int index) {
-        PornhubImageThread t = new PornhubImageThread(url, index, this.workingDir);
+        PornhubImageThread t = new PornhubImageThread(url, index, this.workingDir.toPath());
        pornhubThreadPool.addThread(t);
        try {
            Thread.sleep(IMAGE_SLEEP_TIME);
@@ -130,7 +130,7 @@ public class PornhubRipper extends AbstractHTMLRipper {
        private URL url;
        private int index;

-        PornhubImageThread(URL url, int index, File workingDir) {
+        PornhubImageThread(URL url, int index, Path workingDir) {
            super();
            this.url = url;
            this.index = index;
--- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java
+++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java
@@ -1,10 +1,12 @@
 package com.rarchives.ripme.ripper.rippers;

-import java.io.File;
-import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Date;
 import java.util.List;
 import java.util.regex.Matcher;
@@ -13,6 +15,7 @@ import java.util.regex.Pattern;
 import com.rarchives.ripme.ui.RipStatusMessage;
 import j2html.TagCreator;
 import j2html.tags.ContainerTag;
+import j2html.tags.specialized.DivTag;
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
@@ -241,7 +244,7 @@ public class RedditRipper extends AlbumRipper {
    }

    private void saveText(JSONArray jsonArray) throws JSONException {
-        File saveFileAs;
+        Path saveFileAs;

        JSONObject selfPost = jsonArray.getJSONObject(0).getJSONObject("data")
                .getJSONArray("children").getJSONObject(0).getJSONObject("data");
@@ -284,11 +287,11 @@ public class RedditRipper extends AlbumRipper {
        ).renderFormatted();

        try {
-            saveFileAs = new File(workingDir.getCanonicalPath()
-                    + "" + File.separator
+            saveFileAs = Paths.get(workingDir
+                    + "/"
                    + id + "_" + title.replaceAll("[\\\\/:*?\"<>|]", "")
                    + ".html");
-            FileOutputStream out = new FileOutputStream(saveFileAs);
+            OutputStream out = Files.newOutputStream(saveFileAs);
            out.write(html.getBytes());
            out.close();
        } catch (IOException e) {
@@ -298,26 +301,30 @@ public class RedditRipper extends AlbumRipper {

        LOGGER.debug("Downloading " + url + "'s self post to " + saveFileAs);
        super.retrievingSource(permalink);
-        if (!saveFileAs.getParentFile().exists()) {
-            LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent()));
-            saveFileAs.getParentFile().mkdirs();
+        if (!Files.exists(saveFileAs.getParent())) {
+            LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent().toFile()));
+            try {
+                Files.createDirectory(saveFileAs.getParent());
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
        }
    }

    private ContainerTag getComments(JSONArray comments, String author) {
-        ContainerTag commentsDiv = div().withId("comments");
+        ContainerTag<DivTag> commentsDiv = div().withId("comments");

        for (int i = 0; i < comments.length(); i++) {
            JSONObject data = comments.getJSONObject(i).getJSONObject("data");

-            ContainerTag commentDiv =
+            ContainerTag<DivTag> commentDiv =
                    div(
                            span(data.getString("author")).withClasses("author", iff(data.getString("author").equals(author), "op")),
                            a(new Date((long) data.getInt("created") * 1000).toString()).withHref("#" + data.getString("name"))
                    ).withClass("thing comment").withId(data.getString("name"))
                            .with(rawHtml(Jsoup.parse(data.getString("body_html")).text()));

-            commentDiv = getNestedComments(data, commentDiv, author);
+            getNestedComments(data, commentDiv, author);
            commentsDiv.with(commentDiv);
        }
        return commentsDiv;
@@ -331,7 +338,7 @@ public class RedditRipper extends AlbumRipper {
                        .getJSONArray("children")
                        .getJSONObject(i).getJSONObject("data");

-                ContainerTag childDiv =
+                ContainerTag<DivTag> childDiv =
                        div(
                                div(
                                        span(nestedComment.getString("author")).withClasses("author", iff(nestedComment.getString("author").equals(author), "op")),
@@ -347,7 +354,7 @@ public class RedditRipper extends AlbumRipper {
    }

    private URL parseRedditVideoMPD(String vidURL) {
-        org.jsoup.nodes.Document doc = null;
+        org.jsoup.nodes.Document doc;
        try {
            doc = Http.url(vidURL + "/DASHPlaylist.mpd").ignoreContentType().get();
            int largestHeight = 0;
@@ -395,17 +402,17 @@ public class RedditRipper extends AlbumRipper {
            Matcher m = p.matcher(url);
            if (m.matches()) {
                // It's from reddituploads. Assume .jpg extension.
-                String savePath = this.workingDir + File.separator;
+                String savePath = this.workingDir + "/";
                savePath += id + "-" + m.group(1) + title + ".jpg";
-                addURLToDownload(urls.get(0), new File(savePath));
+                addURLToDownload(urls.get(0), Paths.get(savePath).toFile());
            }
            if (url.contains("v.redd.it")) {
-                String savePath = this.workingDir + File.separator;
+                String savePath = this.workingDir + "/";
                savePath += id + "-" + url.split("/")[3] + title + ".mp4";
                URL urlToDownload = parseRedditVideoMPD(urls.get(0).toExternalForm());
                if (urlToDownload != null) {
                    LOGGER.info("url: " + urlToDownload + " file: " + savePath);
-                    addURLToDownload(urlToDownload, new File(savePath));
+                    addURLToDownload(urlToDownload, Paths.get(savePath).toFile());
                }
            }
            else {
@@ -428,7 +435,6 @@ public class RedditRipper extends AlbumRipper {
        if (Utils.getConfigBoolean("reddit.use_sub_dirs", true)) {
            if (Utils.getConfigBoolean("album_titles.save", true)) {
                subdirectory = title;
-                title = "-" + title + "-";
            }
        }
        for (int i = 0; i < data.length(); i++) {