From 788e0b14e81a65b77d82d687fdc07cc437a99f85 Mon Sep 17 00:00:00 2001 From: Jesse Bate Date: Tue, 4 Jun 2019 22:10:58 +1000 Subject: [PATCH 1/2] Added method to determine best quality image for download [#1324] --- .../ripme/ripper/rippers/FlickrRipper.java | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java index 6ad75003..20514bb9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java @@ -220,18 +220,9 @@ public class FlickrRipper extends AbstractHTMLRipper { for (int i = 0; i < pictures.length(); i++) { LOGGER.info(i); JSONObject data = (JSONObject) pictures.get(i); - // TODO this is a total hack, we should loop over all image sizes and pick the biggest one and not - // just assume - List imageSizes = Arrays.asList("k", "h", "l", "n", "c", "z", "t"); - for ( String imageSize : imageSizes) { - try { - addURLToDownload(new URL(data.getString("url_" + imageSize))); - LOGGER.info("Adding picture " + data.getString("url_" + imageSize)); - break; - } catch (org.json.JSONException ignore) { - // TODO warn the user when we hit a Malformed url - } catch (MalformedURLException e) {} - } + try { + addURLToDownload(getLargestImageURL(data.getString("id"), getAPIKey(doc))); + } catch (IOException ignore) { } } if (x >= totalPages) { // The rips done @@ -250,4 +241,22 @@ public class FlickrRipper extends AbstractHTMLRipper { public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index)); } + + private URL getLargestImageURL(String imageID, String apiKey) throws IOException { + URL imageAPIURL = new URL("https://www.flickr.com/services/rest/?method=flickr.photos.getSizes&api_key=" + apiKey + "&photo_id=" + imageID + "&format=json&nojsoncallback=1"); + TreeMap imageURLMap = new TreeMap<>(); + + try { + JSONArray imageSizes = new JSONObject(Http.url(imageAPIURL).ignoreContentType().get().text()).getJSONObject("sizes").getJSONArray("size"); + for (int i = 0; i < imageSizes.length(); i++) { + JSONObject imageInfo = imageSizes.getJSONObject(i); + imageURLMap.put(imageInfo.getInt("width") * imageInfo.getInt("height"), imageInfo.getString("source")); + } + + } catch (org.json.JSONException ignore) { + + } catch (MalformedURLException e) {} + + return new URL(imageURLMap.lastEntry().getValue()); + } } From 81293b53ed1bfbf5e5d984ec7364c676ebe751cb Mon Sep 17 00:00:00 2001 From: Jesse Bate Date: Thu, 20 Jun 2019 18:38:45 +1000 Subject: [PATCH 2/2] Added error logging instead of throwing away errors blindly. --- .../ripme/ripper/rippers/FlickrRipper.java | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java index 20514bb9..e56cb4a1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java @@ -207,10 +207,10 @@ public class FlickrRipper extends AbstractHTMLRipper { @Override public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList<>(); - + String apiKey = getAPIKey(doc); int x = 1; while (true) { - JSONObject jsonData = getJSON(String.valueOf(x), getAPIKey(doc)); + JSONObject jsonData = getJSON(String.valueOf(x), apiKey); if (jsonData.has("stat") && jsonData.getString("stat").equals("fail")) { break; } else { @@ -221,8 +221,11 @@ public class FlickrRipper extends AbstractHTMLRipper { LOGGER.info(i); JSONObject data = (JSONObject) pictures.get(i); try { - addURLToDownload(getLargestImageURL(data.getString("id"), getAPIKey(doc))); - } catch (IOException ignore) { } + addURLToDownload(getLargestImageURL(data.getString("id"), apiKey)); + } catch (MalformedURLException e) { + LOGGER.error("Flickr MalformedURLException: " + e.getMessage()); + } + } if (x >= totalPages) { // The rips done @@ -242,20 +245,24 @@ public class FlickrRipper extends AbstractHTMLRipper { addURLToDownload(url, getPrefix(index)); } - private URL getLargestImageURL(String imageID, String apiKey) throws IOException { - URL imageAPIURL = new URL("https://www.flickr.com/services/rest/?method=flickr.photos.getSizes&api_key=" + apiKey + "&photo_id=" + imageID + "&format=json&nojsoncallback=1"); + private URL getLargestImageURL(String imageID, String apiKey) throws MalformedURLException { TreeMap imageURLMap = new TreeMap<>(); try { + URL imageAPIURL = new URL("https://www.flickr.com/services/rest/?method=flickr.photos.getSizes&api_key=" + apiKey + "&photo_id=" + imageID + "&format=json&nojsoncallback=1"); JSONArray imageSizes = new JSONObject(Http.url(imageAPIURL).ignoreContentType().get().text()).getJSONObject("sizes").getJSONArray("size"); for (int i = 0; i < imageSizes.length(); i++) { JSONObject imageInfo = imageSizes.getJSONObject(i); imageURLMap.put(imageInfo.getInt("width") * imageInfo.getInt("height"), imageInfo.getString("source")); } - } catch (org.json.JSONException ignore) { - - } catch (MalformedURLException e) {} + } catch (org.json.JSONException e) { + LOGGER.error("Error in parsing of Flickr API: " + e.getMessage()); + } catch (MalformedURLException e) { + LOGGER.error("Malformed URL returned by API"); + } catch (IOException e) { + LOGGER.error("IOException while looking at image sizes: " + e.getMessage()); + } return new URL(imageURLMap.lastEntry().getValue()); }