mirror/ripme
1
0
Fork 0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-08-17 19:26:34 +02:00
Code Issues Releases Wiki Activity

multiple rippers java.nio

Browse Source
This commit is contained in:
soloturn
2022-01-03 16:29:19 +01:00
parent 627152853e
commit e3f6499840
5 changed files with 59 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java
View File

@@ -1,10 +1,11 @@
package com.rarchives.ripme.ripper.rippers; package com.rarchives.ripme.ripper.rippers;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
@@ -330,9 +331,9 @@ public class FivehundredpxRipper extends AbstractJSONRipper {
public void downloadURL(URL url, int index) { public void downloadURL(URL url, int index) {
String u = url.toExternalForm(); String u = url.toExternalForm();
String[] fields = u.split("/"); String[] fields = u.split("/");
String prefix = getPrefix(index) + fields[fields.length - 3]; String prefix = "/" + getPrefix(index) + fields[fields.length - 3];
File saveAs = new File(getWorkingDir() + File.separator + prefix + ".jpg"); Path saveAs = Paths.get(getWorkingDir() + prefix + ".jpg");
addURLToDownload(url, saveAs, "", null, false); addURLToDownload(url, saveAs.toFile(), "", null, false);
} }
} }

36
src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java
View File

@@ -1,10 +1,12 @@
package com.rarchives.ripme.ripper.rippers; package com.rarchives.ripme.ripper.rippers;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@@ -15,7 +17,6 @@ import java.util.regex.Pattern;
import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage;
import com.rarchives.ripme.utils.Utils; import com.rarchives.ripme.utils.Utils;
import org.jsoup.Connection.Response; import org.jsoup.Connection.Response;
import org.jsoup.HttpStatusException;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
@@ -91,14 +92,13 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
String nextUrl = urlBase + nextPageUrl.first().attr("href"); String nextUrl = urlBase + nextPageUrl.first().attr("href");
sleep(500); sleep(500);
Document nextPage = Http.url(nextUrl).cookies(cookies).get();
return nextPage; return Http.url(nextUrl).cookies(cookies).get();
} }
private String getImageFromPost(String url) { private String getImageFromPost(String url) {
sleep(1000); sleep(1000);
Document d = null; Document d;
try { try {
d = Http.url(url).cookies(cookies).get(); d = Http.url(url).cookies(cookies).get();
Elements links = d.getElementsByTag("a"); Elements links = d.getElementsByTag("a");
@@ -184,24 +184,22 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
} }
String newText = ""; String newText = "";
String saveAs = ""; String saveAs = "";
File saveFileAs; Path saveFileAs;
saveAs = text.split("\n")[0]; saveAs = text.split("\n")[0];
saveAs = saveAs.replaceAll("^(\\S+)\\s+by\\s+(.*)$", "$2_$1"); saveAs = saveAs.replaceAll("^(\\S+)\\s+by\\s+(.*)$", "$2_$1");
for (int i = 1;i < text.split("\n").length; i++) { for (int i = 1;i < text.split("\n").length; i++) {
newText = newText.replace("\\","").replace("/","").replace("~","") + "\n" + text.split("\n")[i]; newText = newText.replace("\\","").replace("/","").replace("~","") + "\n" + text.split("\n")[i];
} }
try { try {
if (!subdirectory.equals("")) { saveFileAs = Paths.get(
subdirectory = File.separator + subdirectory; workingDir
} + "/"
saveFileAs = new File(
workingDir.getCanonicalPath()
+ subdirectory + subdirectory
+ File.separator + "/"
+ saveAs + saveAs
+ ".txt"); + ".txt");
// Write the file // Write the file
FileOutputStream out = (new FileOutputStream(saveFileAs)); OutputStream out = Files.newOutputStream(saveFileAs);
out.write(text.getBytes()); out.write(text.getBytes());
out.close(); out.close();
} catch (IOException e) { } catch (IOException e) {
@@ -209,9 +207,13 @@ public class FuraffinityRipper extends AbstractHTMLRipper {
return false; return false;
} }
LOGGER.debug("Downloading " + url + "'s description to " + saveFileAs); LOGGER.debug("Downloading " + url + "'s description to " + saveFileAs);
if (!saveFileAs.getParentFile().exists()) { if (!Files.exists(saveFileAs.getParent())) {
LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent().toFile()));
saveFileAs.getParentFile().mkdirs(); try {
Files.createDirectory(saveFileAs.getParent());
} catch (IOException e) {
e.printStackTrace();
}
} }
return true; return true;
} }

13
src/main/java/com/rarchives/ripme/ripper/rippers/PahealRipper.java
View File

@@ -3,12 +3,13 @@ package com.rarchives.ripme.ripper.rippers;
import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.Utils; import com.rarchives.ripme.utils.Utils;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.net.URL; import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@@ -58,7 +59,7 @@ public class PahealRipper extends AbstractHTMLRipper {
@Override @Override
public Document getNextPage(Document page) throws IOException { public Document getNextPage(Document page) throws IOException {
for (Element e : page.select("#paginator a")) { for (Element e : page.select("#paginator a")) {
if (e.text().toLowerCase().equals("next")) { if (e.text().equalsIgnoreCase("next")) {
return Http.url(e.absUrl("href")).cookies(getCookies()).get(); return Http.url(e.absUrl("href")).cookies(getCookies()).get();
} }
} }
@@ -90,12 +91,12 @@ public class PahealRipper extends AbstractHTMLRipper {
name = name.substring(0, name.length() - ext.length()); name = name.substring(0, name.length() - ext.length());
} }
File outFile = new File(workingDir.getCanonicalPath() Path outFile = Paths.get(workingDir
+ File.separator + "/"
+ Utils.filesystemSafe(new URI(name).getPath()) + Utils.filesystemSafe(new URI(name).getPath())
+ ext); + ext);
addURLToDownload(url, outFile); addURLToDownload(url, outFile.toFile());
} catch (IOException | URISyntaxException ex) { } catch (URISyntaxException ex) {
logger.error("Error while downloading URL " + url, ex); logger.error("Error while downloading URL " + url, ex);
} }
} }

6
src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java
View File

@@ -1,9 +1,9 @@
package com.rarchives.ripme.ripper.rippers; package com.rarchives.ripme.ripper.rippers;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.nio.file.Path;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
@@ -74,7 +74,7 @@ public class PornhubRipper extends AbstractHTMLRipper {
@Override @Override
protected void downloadURL(URL url, int index) { protected void downloadURL(URL url, int index) {
PornhubImageThread t = new PornhubImageThread(url, index, this.workingDir); PornhubImageThread t = new PornhubImageThread(url, index, this.workingDir.toPath());
pornhubThreadPool.addThread(t); pornhubThreadPool.addThread(t);
try { try {
Thread.sleep(IMAGE_SLEEP_TIME); Thread.sleep(IMAGE_SLEEP_TIME);
@@ -130,7 +130,7 @@ public class PornhubRipper extends AbstractHTMLRipper {
private URL url; private URL url;
private int index; private int index;
PornhubImageThread(URL url, int index, File workingDir) { PornhubImageThread(URL url, int index, Path workingDir) {
super(); super();
this.url = url; this.url = url;
this.index = index; this.index = index;

44
src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java
View File

@@ -1,10 +1,12 @@
package com.rarchives.ripme.ripper.rippers; package com.rarchives.ripme.ripper.rippers;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
@@ -13,6 +15,7 @@ import java.util.regex.Pattern;
import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage;
import j2html.TagCreator; import j2html.TagCreator;
import j2html.tags.ContainerTag; import j2html.tags.ContainerTag;
import j2html.tags.specialized.DivTag;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
@@ -241,7 +244,7 @@ public class RedditRipper extends AlbumRipper {
} }
private void saveText(JSONArray jsonArray) throws JSONException { private void saveText(JSONArray jsonArray) throws JSONException {
File saveFileAs; Path saveFileAs;
JSONObject selfPost = jsonArray.getJSONObject(0).getJSONObject("data") JSONObject selfPost = jsonArray.getJSONObject(0).getJSONObject("data")
.getJSONArray("children").getJSONObject(0).getJSONObject("data"); .getJSONArray("children").getJSONObject(0).getJSONObject("data");
@@ -284,11 +287,11 @@ public class RedditRipper extends AlbumRipper {
).renderFormatted(); ).renderFormatted();
try { try {
saveFileAs = new File(workingDir.getCanonicalPath() saveFileAs = Paths.get(workingDir
+ "" + File.separator + "/"
+ id + "_" + title.replaceAll("[\\\\/:*?\"<>|]", "") + id + "_" + title.replaceAll("[\\\\/:*?\"<>|]", "")
+ ".html"); + ".html");
FileOutputStream out = new FileOutputStream(saveFileAs); OutputStream out = Files.newOutputStream(saveFileAs);
out.write(html.getBytes()); out.write(html.getBytes());
out.close(); out.close();
} catch (IOException e) { } catch (IOException e) {
@@ -298,26 +301,30 @@ public class RedditRipper extends AlbumRipper {
LOGGER.debug("Downloading " + url + "'s self post to " + saveFileAs); LOGGER.debug("Downloading " + url + "'s self post to " + saveFileAs);
super.retrievingSource(permalink); super.retrievingSource(permalink);
if (!saveFileAs.getParentFile().exists()) { if (!Files.exists(saveFileAs.getParent())) {
LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent().toFile()));
saveFileAs.getParentFile().mkdirs(); try {
Files.createDirectory(saveFileAs.getParent());
} catch (IOException e) {
e.printStackTrace();
}
} }
} }
private ContainerTag getComments(JSONArray comments, String author) { private ContainerTag getComments(JSONArray comments, String author) {
ContainerTag commentsDiv = div().withId("comments"); ContainerTag<DivTag> commentsDiv = div().withId("comments");
for (int i = 0; i < comments.length(); i++) { for (int i = 0; i < comments.length(); i++) {
JSONObject data = comments.getJSONObject(i).getJSONObject("data"); JSONObject data = comments.getJSONObject(i).getJSONObject("data");
ContainerTag commentDiv = ContainerTag<DivTag> commentDiv =
div( div(
span(data.getString("author")).withClasses("author", iff(data.getString("author").equals(author), "op")), span(data.getString("author")).withClasses("author", iff(data.getString("author").equals(author), "op")),
a(new Date((long) data.getInt("created") * 1000).toString()).withHref("#" + data.getString("name")) a(new Date((long) data.getInt("created") * 1000).toString()).withHref("#" + data.getString("name"))
).withClass("thing comment").withId(data.getString("name")) ).withClass("thing comment").withId(data.getString("name"))
.with(rawHtml(Jsoup.parse(data.getString("body_html")).text())); .with(rawHtml(Jsoup.parse(data.getString("body_html")).text()));
commentDiv = getNestedComments(data, commentDiv, author); getNestedComments(data, commentDiv, author);
commentsDiv.with(commentDiv); commentsDiv.with(commentDiv);
} }
return commentsDiv; return commentsDiv;
@@ -331,7 +338,7 @@ public class RedditRipper extends AlbumRipper {
.getJSONArray("children") .getJSONArray("children")
.getJSONObject(i).getJSONObject("data"); .getJSONObject(i).getJSONObject("data");
ContainerTag childDiv = ContainerTag<DivTag> childDiv =
div( div(
div( div(
span(nestedComment.getString("author")).withClasses("author", iff(nestedComment.getString("author").equals(author), "op")), span(nestedComment.getString("author")).withClasses("author", iff(nestedComment.getString("author").equals(author), "op")),
@@ -347,7 +354,7 @@ public class RedditRipper extends AlbumRipper {
} }
private URL parseRedditVideoMPD(String vidURL) { private URL parseRedditVideoMPD(String vidURL) {
org.jsoup.nodes.Document doc = null; org.jsoup.nodes.Document doc;
try { try {
doc = Http.url(vidURL + "/DASHPlaylist.mpd").ignoreContentType().get(); doc = Http.url(vidURL + "/DASHPlaylist.mpd").ignoreContentType().get();
int largestHeight = 0; int largestHeight = 0;
@@ -395,17 +402,17 @@ public class RedditRipper extends AlbumRipper {
Matcher m = p.matcher(url); Matcher m = p.matcher(url);
if (m.matches()) { if (m.matches()) {
// It's from reddituploads. Assume .jpg extension. // It's from reddituploads. Assume .jpg extension.
String savePath = this.workingDir + File.separator; String savePath = this.workingDir + "/";
savePath += id + "-" + m.group(1) + title + ".jpg"; savePath += id + "-" + m.group(1) + title + ".jpg";
addURLToDownload(urls.get(0), new File(savePath)); addURLToDownload(urls.get(0), Paths.get(savePath).toFile());
} }
if (url.contains("v.redd.it")) { if (url.contains("v.redd.it")) {
String savePath = this.workingDir + File.separator; String savePath = this.workingDir + "/";
savePath += id + "-" + url.split("/")[3] + title + ".mp4"; savePath += id + "-" + url.split("/")[3] + title + ".mp4";
URL urlToDownload = parseRedditVideoMPD(urls.get(0).toExternalForm()); URL urlToDownload = parseRedditVideoMPD(urls.get(0).toExternalForm());
if (urlToDownload != null) { if (urlToDownload != null) {
LOGGER.info("url: " + urlToDownload + " file: " + savePath); LOGGER.info("url: " + urlToDownload + " file: " + savePath);
addURLToDownload(urlToDownload, new File(savePath)); addURLToDownload(urlToDownload, Paths.get(savePath).toFile());
} }
} }
else { else {
@@ -428,7 +435,6 @@ public class RedditRipper extends AlbumRipper {
if (Utils.getConfigBoolean("reddit.use_sub_dirs", true)) { if (Utils.getConfigBoolean("reddit.use_sub_dirs", true)) {
if (Utils.getConfigBoolean("album_titles.save", true)) { if (Utils.getConfigBoolean("album_titles.save", true)) {
subdirectory = title; subdirectory = title;
title = "-" + title + "-";
} }
} }
for (int i = 0; i < data.length(); i++) { for (int i = 0; i < data.length(); i++) {