mirror/ripme
1
0
Fork 0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-08-26 15:24:51 +02:00
Code Issues Releases Wiki Activity

Fix Flaky Tests

Browse Source 
stop() is called too soon in some cases, download might not have been started
and calling stop will leave the items in pending
so only call stop if items have been completed or on error
+ improve shouldStop as AtomicBoolean
+ trace logging in case issue happens again
This commit is contained in:
Stilgar
2021-05-21 00:46:46 +02:00
parent 3c4e303358
commit 05f97cfadc
5 changed files with 25 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java
View File

@@ -128,7 +128,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
index += 1;
LOGGER.debug("Found image url #" + index + ": " + imageURL);
downloadURL(new URL(imageURL), index);
if (isStopped()) {
if (isStopped() || isThisATest()) {
break;
}
}
@@ -139,7 +139,7 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
if (!textURLs.isEmpty()) {
LOGGER.debug("Found description link(s) from " + doc.location());
for (String textURL : textURLs) {
if (isStopped()) {
if (isStopped() || isThisATest()) {
break;
}
textindex += 1;
@@ -294,9 +294,9 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
*/
public boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies, Boolean getFileExtFromMIME) {
// Only download one file if this is a test.
if (super.isThisATest() &&
(itemsPending.size() > 0 || itemsCompleted.size() > 0 || itemsErrored.size() > 0)) {
if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) {
stop();
itemsPending.clear();
return false;
}
if (!allowDuplicates()

4
src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java
View File

@@ -142,9 +142,9 @@ public abstract class AbstractJSONRipper extends AbstractRipper {
*/
public boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies, Boolean getFileExtFromMIME) {
// Only download one file if this is a test.
if (super.isThisATest() &&
(itemsPending.size() > 0 || itemsCompleted.size() > 0 || itemsErrored.size() > 0)) {
if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) {
stop();
itemsPending.clear();
return false;
}
if (!allowDuplicates()

11
src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
View File

@@ -14,6 +14,8 @@ import java.util.List;
import java.util.Map;
import java.util.Observable;
import java.util.Scanner;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.log4j.FileAppender;
import org.apache.log4j.Logger;
import org.jsoup.HttpStatusException;
@@ -47,17 +49,18 @@ public abstract class AbstractRipper
public boolean hasASAPRipping() { return false; }
// Everytime addUrlToDownload skips a already downloaded url this increases by 1
public int alreadyDownloadedUrls = 0;
private boolean shouldStop = false;
private final AtomicBoolean shouldStop = new AtomicBoolean(false);
private static boolean thisIsATest = false;
public void stop() {
shouldStop = true;
LOGGER.trace("stop()");
shouldStop.set(true);
}
public boolean isStopped() {
return shouldStop;
return shouldStop.get();
}
protected void stopCheck() throws IOException {
if (shouldStop) {
if (shouldStop.get()) {
throw new IOException("Ripping interrupted");
}
}

4
src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java
View File

@@ -52,9 +52,9 @@ public abstract class AlbumRipper extends AbstractRipper {
*/
public boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies, Boolean getFileExtFromMIME) {
// Only download one file if this is a test.
if (super.isThisATest() &&
(itemsPending.size() > 0 || itemsCompleted.size() > 0 || itemsErrored.size() > 0)) {
if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) {
stop();
itemsPending.clear();
return false;
}
if (!allowDuplicates()

7
src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java
View File

@@ -32,6 +32,13 @@ public class RippersTest {
ripper.setup();
ripper.markAsTest();
ripper.rip();
if (logger.isTraceEnabled()) {
logger.trace("working dir: " + ripper.getWorkingDir());
logger.trace("list files: " + ripper.getWorkingDir().listFiles().length);
for (int i = 0; i < ripper.getWorkingDir().listFiles().length; i++) {
logger.trace(" " + ripper.getWorkingDir().listFiles()[i]);
}
}
Assertions.assertTrue(ripper.getWorkingDir().listFiles().length >= 1,
"Failed to download a single file from " + ripper.getURL());
} catch (IOException e) {