Cache validation. Needs a DataLoader (itunes is 1.5mb request)

src/infoplugins/generic/newreleases/NewReleasesPlugin.cpp

@@ -1,3 +1,24 @@
+/* === This file is part of Tomahawk Player - <http://tomahawk-player.org> ===
+ *
+ *   Copyright 2012, Casey Link <unnamedrambler@gmail.com>
+ *   Copyright 2011-2012, Hugo Lindström <hugolm84@gmail.com>
+ *   Copyright 2011, Leo Franchi <lfranchi@kde.org>
+ *   Copyright 2010-2011, Jeff Mitchell <jeff@tomahawk-player.org>
+ *
+ *   Tomahawk is free software: you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, either version 3 of the License, or
+ *   (at your option) any later version.
+ *
+ *   Tomahawk is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with Tomahawk. If not, see <http://www.gnu.org/licenses/>.
+ */
+
 #include "NewReleasesPlugin.h"
 
 #include <QtCore/QDir>
@@ -15,11 +36,10 @@
 #include "utils/Logger.h"
 #include "utils/TomahawkCache.h"
 #include "Source.h"
-
 #include <qjson/parser.h>
 #include <qjson/serializer.h>
 
-#define CHART_URL "http://charts.tomahawk-player.org/"
+#define CHART_URL "http://charts.stage.tomahawk-player.org/"
 //#define CHART_URL "http://localhost:8080/"
 
 using namespace Tomahawk::InfoSystem;
@@ -42,7 +62,7 @@ newReleaseSort( const InfoStringHash& left, const InfoStringHash& right )
 
 NewReleasesPlugin::NewReleasesPlugin()
     : InfoPlugin()
-    , m_nrFetchJobs ( 0 )
+    , m_nrFetchJobs( 0 )
 {
     m_nrVersion = "0.5";
     m_supportedGetTypes << InfoNewReleaseCapabilities << InfoNewRelease;
@@ -51,30 +71,46 @@ NewReleasesPlugin::NewReleasesPlugin()
 
 NewReleasesPlugin::~NewReleasesPlugin()
 {
-    tDebug ( LOGVERBOSE ) << Q_FUNC_INFO;
+    tDebug( LOGVERBOSE ) << Q_FUNC_INFO;
 }
 
-
+/**
+ * @brief NewReleasesPlugin::init
+ * Loops through cache expiration timestamps
+ * Refetches source/items if invalid
+ */
 void
 NewReleasesPlugin::init()
 {
-    QVariantList source_qvarlist = TomahawkUtils::Cache::instance()->getData( "NewReleasesPlugin", "nr_sources" ).toList();
+    QVariant data = TomahawkUtils::Cache::instance()->getData( "NewReleasesPlugin", "nr_sources" );
-    foreach( const QVariant & source, source_qvarlist )
+    if ( data.canConvert< QList< Tomahawk::InfoSystem::InfoStringHash > >() )
     {
-        m_nrSources.append( source.toString() );
+         const QList< Tomahawk::InfoSystem::InfoStringHash > sourceList = data.value< QList< Tomahawk::InfoSystem::InfoStringHash > >();
-        tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "fetched source from cache" << source.toString();
+         foreach ( const Tomahawk::InfoSystem::InfoStringHash &sourceHash, sourceList )
-
+         {
+             bool ok;
+             qlonglong maxAge = getMaxAge( QString(sourceHash[ "nr_expires" ]).toLongLong( &ok ) );
+             if ( !ok || maxAge <= 0 )
+             {
+                 // This source has expired.
+                 m_refetchSource << sourceHash[ "nr_source" ];
+             }
+             m_nrSources << sourceHash;
+         }
     }
-    tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "total sources" << m_nrSources.size() << source_qvarlist.size();
-    if( m_nrSources.size() == 0 )
-        fetchNRSourcesList( true );
-}
 
+    tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "total sources" << m_nrSources.size() << m_nrSources;
+
+    if( m_nrSources.size() == 0 || m_refetchSource.size() != 0 )
+    {
+        tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Will refetch on next request. Empty or Invalid CACHE" << m_nrSources.size() << m_refetchSource;
+    }
+}
 
 void
 NewReleasesPlugin::dataError( InfoRequestData requestData )
 {
-    emit info ( requestData, QVariant() );
+    emit info( requestData, QVariant() );
     return;
 }
 
@@ -82,27 +118,25 @@ NewReleasesPlugin::dataError( InfoRequestData requestData )
 void
 NewReleasesPlugin::getInfo( InfoRequestData requestData )
 {
-    //qDebug() << Q_FUNC_INFO << requestData.caller;
-    //qDebug() << Q_FUNC_INFO << requestData.customData;
 
     InfoStringHash hash = requestData.input.value< Tomahawk::InfoSystem::InfoStringHash >();
     bool foundSource = false;
 
-    switch ( requestData.type )
+    switch( requestData.type )
     {
     case InfoNewRelease:
         /// We need something to check if the request is actually ment to go to this plugin
-        if ( !hash.contains ( "nr_source" ) )
+        if ( !hash.contains( "nr_source" ) )
         {
-            tDebug ( LOGVERBOSE ) << Q_FUNC_INFO << "Hash did not contain required param!";
+            tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Hash did not contain required param!";
-            dataError ( requestData );
+            dataError( requestData );
             break;
         }
         else
         {
-            foreach ( QString resource, m_nrSources )
+            foreach ( const Tomahawk::InfoSystem::InfoStringHash &sourceHash, m_nrSources )
             {
-                if ( resource == hash["nr_source"] )
+                if ( sourceHash[ "nr_source" ] == hash[ "nr_source" ] )
                 {
                     foundSource = true;
                 }
@@ -110,20 +144,22 @@ NewReleasesPlugin::getInfo( InfoRequestData requestData )
 
             if ( !foundSource )
             {
+                tDebug ( LOGVERBOSE ) << Q_FUNC_INFO << "Hash did not contain source " << hash["nr_source"];
                 dataError ( requestData );
                 break;
             }
 
         }
-        fetchNRFromCache ( requestData );
+        fetchNRFromCache( requestData );
         break;
 
     case InfoNewReleaseCapabilities:
-        fetchNRCapabilitiesFromCache ( requestData );
+        tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Requesting InfoNewReleaseCapabilities from cache";
+        fetchNRCapabilitiesFromCache( requestData );
         break;
 
     default:
-        dataError ( requestData );
+        dataError( requestData );
     }
 }
 
@@ -133,26 +169,38 @@ NewReleasesPlugin::fetchNRFromCache( InfoRequestData requestData )
 {
     if ( !requestData.input.canConvert< Tomahawk::InfoSystem::InfoStringHash >() )
     {
-        dataError ( requestData );
+        tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Hash did not contain source " << requestData.input;
+        dataError( requestData );
         return;
     }
 
     InfoStringHash hash = requestData.input.value< Tomahawk::InfoSystem::InfoStringHash >();
     Tomahawk::InfoSystem::InfoStringHash criteria;
 
-    /// Each request needs to contain both a id and source
+    /// Each request needs to contain both a id, source a expire header
-    if ( !hash.contains ( "nr_id" ) && !hash.contains ( "nr_source" ) )
+    if ( !hash.contains( "nr_id" ) && !hash.contains( "nr_source" ) && !hash.contains( "nr_expires" ) )
     {
-        tDebug ( LOGVERBOSE ) << Q_FUNC_INFO << "Hash did not contain required params!";
+        tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Hash did not contain required params!";
-        dataError ( requestData );
+        dataError( requestData );
         return;
 
     }
     /// Set the criterias for current chart
-    criteria["nr_id"] = hash["nr_id"];
+    criteria[ "nr_id" ] = hash[ "nr_id" ];
-    criteria["nr_source"] = hash["nr_source"];
+    criteria[ "nr_source" ] = hash[ "nr_source" ];
+    criteria[ "nr_expires" ] = hash[ "nr_expires" ];
 
-    emit getCachedInfo ( criteria, 86400000, requestData );
+    bool ok;
+    const qlonglong maxAge = getMaxAge( QString( hash[ "nr_expires" ] ).toLongLong( &ok ) );
+
+    if ( !ok || maxAge <= 0 )
+    {
+        emit notInCacheSlot( criteria, requestData );
+        return;
+    }
+
+    emit getCachedInfo( criteria, maxAge, requestData );
+    return;
 }
 
 
@@ -161,70 +209,84 @@ NewReleasesPlugin::fetchNRCapabilitiesFromCache( InfoRequestData requestData )
 {
     if ( !requestData.input.canConvert< Tomahawk::InfoSystem::InfoStringHash >() )
     {
-        tDebug ( LOGVERBOSE ) << Q_FUNC_INFO << "Could not convert requestData to InfoStringHash!";
+        tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Could not convert requestData to InfoStringHash!";
-        dataError ( requestData );
+        dataError( requestData );
         return;
     }
 
     Tomahawk::InfoSystem::InfoStringHash criteria;
     criteria[ "InfoNewReleaseCapabilities" ] = "newreleasesplugin";
     criteria[ "InfoNewReleaseVersion" ] = m_nrVersion;
-    emit getCachedInfo ( criteria, 864000000, requestData );
+
+    /**
+     * Someone requested capabilities, but init() told us someone was out of date
+     * Next fetch will fetch those that are invalid
+     */
+    if ( m_refetchSource.size() != 0 )
+    {
+        tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Some sources need to refetch!" << m_refetchSource;
+        emit notInCacheSlot( criteria, requestData );
+        return;
+    }
+
+    tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Fetching fetchNRCapabilitiesFromCache";
+    emit getCachedInfo ( criteria, 172800000 /* 2 days */, requestData );
 }
 
 
 void
 NewReleasesPlugin::notInCacheSlot( InfoStringHash criteria, InfoRequestData requestData )
 {
-    switch ( requestData.type )
+    switch( requestData.type )
     {
         case InfoNewRelease:
         {
-        tDebug ( LOGVERBOSE ) << Q_FUNC_INFO << "InfoNewRelease not in cache! Fetching...";
+            tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "InfoNewRelease not in cache! Fetching...";
-        fetchNR ( requestData, criteria["nr_source"], criteria["nr_id"] );
+            fetchNR( requestData, criteria[ "nr_source" ], criteria[ "nr_id" ] );
+            m_cachedRequests.append( requestData );
             return;
-
         }
 
         case InfoNewReleaseCapabilities:
         {
-        tDebug ( LOGVERBOSE ) << Q_FUNC_INFO << "InfoChartCapabilities not in cache! Fetching...";
+            tDebug ( LOGVERBOSE ) << Q_FUNC_INFO << "InfoChartCapabilities not in cache! Fetching..." << criteria << requestData.requestId;
-        fetchNRSourcesList( false );
-        m_cachedRequests.append ( requestData );
 
+            QUrl url = QUrl( QString ( CHART_URL "newreleases" ) );
+            url.addQueryItem( "version", TomahawkUtils::appFriendlyVersion() );
+
+            QNetworkReply* reply = TomahawkUtils::nam()->get( QNetworkRequest ( url ) );
+            reply->setProperty( "only_source_list", true );
+
+            tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "fetching:" << url;
+            connect( reply, SIGNAL ( finished() ), SLOT ( nrSourcesList() ) );
+
+            m_nrFetchJobs++;
+
+            if ( m_nrFetchJobs > 0 )
+            {
+                qDebug() << Q_FUNC_INFO << "InfoChartCapabilities still fetching!";
+                m_cachedRequests.append( requestData );
+                return;
+            }
+
+            emit info( requestData, m_allNRsMap );
             return;
         }
 
         default:
         {
             tLog() << Q_FUNC_INFO << "Couldn't figure out what to do with this type of request after cache miss";
-        emit info ( requestData, QVariant() );
+            emit info( requestData, QVariant() );
             return;
         }
     }
 }
 
-
-void
-NewReleasesPlugin::fetchNRSourcesList( bool fetchOnlySourcesList )
-{
-    QUrl url = QUrl ( QString ( CHART_URL "newreleases" ) );
-    url.addQueryItem( "version", TomahawkUtils::appFriendlyVersion() );
-
-    QNetworkReply* reply = TomahawkUtils::nam()->get ( QNetworkRequest ( url ) );
-    reply->setProperty( "only_source_list", fetchOnlySourcesList );
-
-    tDebug() << "fetching:" << url;
-    connect ( reply, SIGNAL ( finished() ), SLOT ( nrSourcesList() ) );
-
-}
-
-
 void
 NewReleasesPlugin::nrSourcesList()
 {
-    tDebug ( LOGVERBOSE ) << "Got newreleases sources list";
+    tDebug ( LOGVERBOSE ) << Q_FUNC_INFO << "Got newreleases sources list";
-    QNetworkReply* reply = qobject_cast<QNetworkReply*> ( sender() );
+    QNetworkReply* reply = qobject_cast<QNetworkReply*>( sender() );
 
     if ( reply->error() == QNetworkReply::NoError )
     {
@@ -235,37 +297,93 @@ NewReleasesPlugin::nrSourcesList()
 
         if ( !ok )
         {
-            tLog() << "Failed to parse sources" << p.errorString() << "On line" << p.errorLine();
+            tDebug() << Q_FUNC_INFO  << "Failed to parse sources" << p.errorString() << "On line" << p.errorLine();
             return;
         }
 
-        m_nrSources.clear();
+        foreach ( const QVariant &rsource, sources )
-        foreach ( const QVariant &source, sources )
         {
-            m_nrSources << source.toString();
+
+            /**
+             * Get the maxAge for this source, update if invalid
+             * We dont want to refetch all if not necessary
+             */
+            const QString source = rsource.toString();
+
+            if ( !m_refetchSource.contains( source ) && !m_allNRsMap.empty() )
+            {
+                tDebug() << Q_FUNC_INFO  << "Skipping fetch of valid source" << source;
+                continue;
             }
-        TomahawkUtils::Cache::instance()->putData( "NewReleasesPlugin", 172800000 /* 2 days */, "nr_sources", m_nrSources );
+
+            if ( !m_nrSources.isEmpty() )
+            {
+                for ( int i = 0; i < m_nrSources.size(); i++ )
+                {
+                    const Tomahawk::InfoSystem::InfoStringHash &hash = m_nrSources.at( i );
+                    if ( hash[ "nr_source" ] == source )
+                    {
+                        tDebug() << Q_FUNC_INFO  << "Removing invalid source" << source;
+                        m_nrSources.removeAt( i );
+                    }
+                }
+            }
+
+            /**
+             * @brief Expiration
+             * Each item has an expiration, on next request for cache, it will be checked
+             */
+            const QString headerExpiration = reply->rawHeader( QString( source + "Expires" ).toLocal8Bit() );
+            const qlonglong maxAge = getMaxAge( headerExpiration.toLocal8Bit() );
+            const qlonglong expires = headerExpiration.toLongLong(&ok);
+            Tomahawk::InfoSystem::InfoStringHash source_expire;
+
+            if ( ok )
+            {
+                source_expire[ "nr_source" ] = source;
+                source_expire[ "nr_expires" ] = QString::number(expires);
+                m_nrSources << source_expire;
+            }
+
+            if ( maxAge == 0 )
+            {
+                tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "MaxAge for " << source << " is  0. Fetching all";
+                reply->setProperty( "only_source_list", false );
+            }
+
+        }
+
+        /**
+         * We can store the source list for how long as we want
+         * In init, we check expiration for each source, and refetch if invalid
+         * 2 days seems fair enough though
+         */
+        TomahawkUtils::Cache::instance()->putData( "NewReleasesPlugin", 172800000 /* 2 days */, "nr_sources", QVariant::fromValue< QList< Tomahawk::InfoSystem::InfoStringHash > > ( m_nrSources ) );
+        m_nrFetchJobs--;
         if( !reply->property( "only_source_list" ).toBool() )
+        {
+            tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Fetching all sources!";
             fetchAllNRSources();
         }
+    }
 }
 
-
 void
 NewReleasesPlugin::fetchAllNRSources()
 {
     if ( !m_nrSources.isEmpty() && m_allNRsMap.isEmpty() )
     {
-        tDebug ( LOGVERBOSE ) << Q_FUNC_INFO << "InfoNewRelease fetching source data";
+        tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "InfoNewRelease fetching source data";
-        foreach ( QString source, m_nrSources )
+        foreach ( const Tomahawk::InfoSystem::InfoStringHash source, m_nrSources )
         {
-            QUrl url = QUrl ( QString ( CHART_URL "newreleases/%1" ).arg ( source ) );
+            QUrl url = QUrl( QString ( CHART_URL "newreleases/%1" ).arg( source[ "nr_source" ] ) );
             url.addQueryItem( "version", TomahawkUtils::appFriendlyVersion() );
-            QNetworkReply* reply = TomahawkUtils::nam()->get ( QNetworkRequest ( url ) );
-            reply->setProperty ( "nr_source", source );
 
-            tDebug() << "fetching:" << url;
+            QNetworkReply* reply = TomahawkUtils::nam()->get( QNetworkRequest( url ) );
-            connect ( reply, SIGNAL ( finished() ), SLOT ( nrList() ) );
+            reply->setProperty( "nr_source", source[ "nr_source" ] );
+
+            tDebug() << Q_FUNC_INFO << "fetching:" << url;
+            connect( reply, SIGNAL( finished() ), SLOT( nrList() ) );
 
             m_nrFetchJobs++;
         }
@@ -277,46 +395,45 @@ void
 NewReleasesPlugin::fetchNR( InfoRequestData requestData, const QString& source, const QString& nr_id )
 {
     /// Fetch the chart, we need source and id
-    QUrl url = QUrl ( QString ( CHART_URL "newreleases/%1/%2" ).arg ( source ).arg ( nr_id ) );
+    QUrl url = QUrl ( QString ( CHART_URL "newreleases/%1/%2" ).arg( source ).arg( nr_id ) );
     url.addQueryItem( "version", TomahawkUtils::appFriendlyVersion() );
 
-    tDebug ( LOGVERBOSE ) << Q_FUNC_INFO << "fetching: " << url;
+    tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "fetching: " << url;
 
-    QNetworkReply* reply = TomahawkUtils::nam()->get ( QNetworkRequest ( url ) );
+    QNetworkReply* reply = TomahawkUtils::nam()->get( QNetworkRequest( url ) );
-    reply->setProperty ( "requestData", QVariant::fromValue< Tomahawk::InfoSystem::InfoRequestData > ( requestData ) );
+    reply->setProperty( "requestData", QVariant::fromValue< Tomahawk::InfoSystem::InfoRequestData >( requestData ) );
 
-    connect ( reply, SIGNAL ( finished() ), SLOT ( nrReturned() ) );
+    connect ( reply, SIGNAL( finished() ), SLOT( nrReturned() ) );
 }
 
 
 void
 NewReleasesPlugin::nrList()
 {
-    tDebug ( LOGVERBOSE ) << "Got newreleases list result";
+    tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Got newreleases list result";
-    QNetworkReply* reply = qobject_cast<QNetworkReply*> ( sender() );
+    QNetworkReply* reply = qobject_cast<QNetworkReply*>( sender() );
 
     if ( reply->error() == QNetworkReply::NoError )
     {
         QJson::Parser p;
         bool ok;
-        const QVariantMap res = p.parse ( reply, &ok ).toMap();
+        const QVariantMap res = p.parse( reply, &ok ).toMap();
 
         if ( !ok )
         {
             tLog() << "Failed to parse resources" << p.errorString() << "On line" << p.errorLine();
-
             return;
         }
 
         /// Got types, append!
-        const QString source = reply->property ( "nr_source" ).toString();
+        const QString source = reply->property( "nr_source" ).toString();
+        const qlonglong expires = QString( reply->rawHeader( QString( "Expires" ).toLocal8Bit() ) ).toLongLong( &ok );
 
         // We'll populate newreleases with the data from the server
         QVariantMap newreleases;
         QString nrName;
         QStringList defaultChain;
-        // Building:
+
-        // [Source] - New Release
         QList< InfoStringHash > albumNRs;
         QHash< QString, QVariantMap > extraType;
 
@@ -345,7 +462,7 @@ NewReleasesPlugin::nrList()
                         if ( !m_cachedCountries.contains( geo ) )
                         {
                             extra = Tomahawk::CountryUtils::fullCountryFromCode( geo );
-                            if( extra.isEmpty() || extra.isNull() ){
+                            if ( extra.isEmpty() || extra.isNull() ){
                                 qWarning() << "Geo string seems to be off!" << geo;
                                 continue;
                             }
@@ -372,15 +489,20 @@ NewReleasesPlugin::nrList()
                         nrExtraType = nrMap.value( "extra" ).toString();
                     }
 
-                    InfoStringHash c;
+                    InfoStringHash nr;
-                    c[ "id" ] = id;
+                    nr[ "id" ] = id;
-                    c[ "label" ] = name;
+                    nr[ "label" ] = name;
-                    c[ "type" ] = "album";
+                    nr[ "type" ] = "album";
+                    /**
+                     * If this item has expired, set it to 0.
+                     */
+                    nr[ "expires" ] = ( ok ? QString::number (expires ) : QString::number( 0 ) );
                     if ( isDefault )
-                        c[ "default" ] = "true";
+                        nr[ "default" ] = "true";
+
 
                     QList< Tomahawk::InfoSystem::InfoStringHash > extraTypeData = extraType[ nrExtraType ][ extra ].value< QList< Tomahawk::InfoSystem::InfoStringHash > >();
-                    extraTypeData.append( c );
+                    extraTypeData.append( nr );
                     extraType[ nrExtraType ][ extra ] = QVariant::fromValue< QList< Tomahawk::InfoSystem::InfoStringHash > >( extraTypeData );
 
                     if ( isDefault )
@@ -393,11 +515,11 @@ NewReleasesPlugin::nrList()
                 }
             }
 
-            foreach ( const QString& c, extraType.keys() )
+            foreach ( const QString& nr, extraType.keys() )
             {
-                newreleases[ c ] = extraType[ c ];
+                newreleases[ nr ] = extraType[ nr ];
-//                tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Itunes extraType has types:" << c;
             }
+
             if ( source == "itunes" )
             {
                 nrName = "iTunes";
@@ -415,9 +537,10 @@ NewReleasesPlugin::nrList()
                     const QString extra = nrMap.value( "extra" ).toString();
 
                     InfoStringHash nr;
-                    nr["id"] = nrMap.value ( "id" ).toString();
+                    nr[ "id" ] = nrMap.value( "id" ).toString();
-                    nr["label"] = nrMap.value ( "name" ).toString();
+                    nr[ "label" ] = nrMap.value( "name" ).toString();
-                    nr["date"] = nrMap.value ( "date" ).toString();
+                    nr[ "date" ] = nrMap.value( "date" ).toString();
+                    nr[ "expires" ] = ( ok ? QString::number( expires ) : QString::number( 0 ) );
 
                     if ( type == "Album" )
                     {
@@ -425,13 +548,12 @@ NewReleasesPlugin::nrList()
 
                         if( !extra.isEmpty() )
                         {
-//                            qDebug() << "FOUND EXTRA!! " << extra;
                             QList< Tomahawk::InfoSystem::InfoStringHash > extraTypeData = extraType[ extra ][ type ].value< QList< Tomahawk::InfoSystem::InfoStringHash > >();
                             extraTypeData.append( nr );
                             extraType[ extra ][ type ] = QVariant::fromValue< QList< Tomahawk::InfoSystem::InfoStringHash > >( extraTypeData );
                         }
                         else
-                            albumNRs.append ( nr );
+                            albumNRs.append( nr );
                     }
                     else
                     {
@@ -444,12 +566,11 @@ NewReleasesPlugin::nrList()
                 foreach( const QString& c, extraType.keys() )
                 {
                     newreleases[ c ] = extraType[ c ];
-//                    tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "extraType has types:" << c;
                 }
             }
 
             if ( !albumNRs.isEmpty() )
-                newreleases.insert ( tr ( "Albums" ), QVariant::fromValue< QList< Tomahawk::InfoSystem::InfoStringHash > > ( albumNRs ) );
+                newreleases.insert ( tr ( "Albums" ), QVariant::fromValue< QList< Tomahawk::InfoSystem::InfoStringHash > >( albumNRs ) );
 
             /// @note For displaying purposes, upper the first letter
             /// @note Remeber to lower it when fetching this!
@@ -458,8 +579,8 @@ NewReleasesPlugin::nrList()
         }
 
         /// Add the possible charts and its types to breadcrumb
-        tDebug ( LOGVERBOSE ) << Q_FUNC_INFO << "ADDING newrelease TO NRS:" << nrName;
+        tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "ADDING newrelease TO NRS:" << nrName << newreleases;
-        m_allNRsMap.insert ( nrName, QVariant::fromValue< QVariantMap > ( newreleases ) );
+        m_allNRsMap.insert( nrName, QVariant::fromValue< QVariantMap >( newreleases ) );
     }
     else
     {
@@ -467,27 +588,55 @@ NewReleasesPlugin::nrList()
     }
 
     m_nrFetchJobs--;
+
     if ( !m_cachedRequests.isEmpty() && m_nrFetchJobs == 0 )
     {
         foreach ( InfoRequestData request, m_cachedRequests )
         {
-            emit info ( request, m_allNRsMap );
+            emit info( request, m_allNRsMap );
             // update cache
             Tomahawk::InfoSystem::InfoStringHash criteria;
             criteria[ "InfoNewReleaseCapabilities" ] = "newreleasesplugin";
             criteria[ "InfoNewReleaseVersion" ] = m_nrVersion;
-            emit updateCache ( criteria, 864000000, request.type, m_allNRsMap );
+            /**
+             * We can cache it the lot for 2 days, it will be checked on next request
+             */
+            emit updateCache( criteria, 172800000 /* 2 days */, request.type, m_allNRsMap );
         }
         m_cachedRequests.clear();
     }
 }
 
+qlonglong
+NewReleasesPlugin::getMaxAge( const QByteArray &rawHeader ) const
+{
+    bool ok;
+    qlonglong expires = QString( rawHeader ).toLongLong( &ok );
+    if ( ok )
+    {
+        return getMaxAge( expires );
+    }
+    return 0;
+}
+
+qlonglong
+NewReleasesPlugin::getMaxAge( const qlonglong expires ) const
+{
+    qlonglong currentEpoch = QDateTime::currentMSecsSinceEpoch() / 1000;
+    qlonglong expiresInSeconds = expires-currentEpoch;
+
+    if ( expiresInSeconds > 0 )
+    {
+        return ( qlonglong )expiresInSeconds*1000;
+    }
+    return 0;
+}
 
 void
 NewReleasesPlugin::nrReturned()
 {
     /// Chart request returned something! Woho
-    QNetworkReply* reply = qobject_cast<QNetworkReply*> ( sender() );
+    QNetworkReply* reply = qobject_cast<QNetworkReply*>( sender() );
     QVariantMap returnedData;
 
     if ( reply->error() == QNetworkReply::NoError )
@@ -502,9 +651,12 @@ NewReleasesPlugin::nrReturned()
             return;
         }
 
+        const qlonglong maxAge = getMaxAge( reply->rawHeader( QString( "Expires" ).toLocal8Bit() ) );
+        const qlonglong expires = QString( reply->rawHeader( QString( "Expires" ).toLocal8Bit() ) ).toLongLong( &ok );
+
         /// SO we have a result, parse it!
-        QVariantList albumList = res.value ( "list" ).toList();
+        QVariantList albumList = res.value( "list" ).toList();
-        QList< Tomahawk::InfoSystem::InfoStringHash > newreleases;
+        QList< Tomahawk::InfoSystem::InfoStringHash >newreleases;
 
         foreach( const QVariant & albumObj, albumList )
         {
@@ -516,31 +668,35 @@ NewReleasesPlugin::nrReturned()
                 const QString date = albumMap.value("date").toString();
 
                 Tomahawk::InfoSystem::InfoStringHash pair;
-                pair["artist"] = artist;
+                pair[ "artist" ] = artist;
-                pair["album"] = album;
+                pair[ "album" ] = album;
-                pair["date"] = date;
+                pair[ "date" ] = date;
                 newreleases.append( pair );
             }
         }
 
         qSort( newreleases.begin(), newreleases.end(), newReleaseSort );
 
-//        tDebug() << "NewReleasesPlugin:" << "\tgot " << newreleases.size() << " albums";
         returnedData[ "albums" ] = QVariant::fromValue< QList< Tomahawk::InfoSystem::InfoStringHash > >( newreleases );
         returnedData[ "type" ] = "albums";
         Tomahawk::InfoSystem::InfoRequestData requestData = reply->property( "requestData" ).value< Tomahawk::InfoSystem::InfoRequestData >();
 
         emit info( requestData, returnedData );
+
         // update cache
         Tomahawk::InfoSystem::InfoStringHash criteria;
         Tomahawk::InfoSystem::InfoStringHash origData = requestData.input.value< Tomahawk::InfoSystem::InfoStringHash >();
         criteria[ "nr_id" ] = origData[ "nr_id" ];
         criteria[ "nr_source" ] = origData[ "nr_source" ];
-        emit updateCache( criteria, 86400000, requestData.type, returnedData );
+        criteria[ "nr_expires" ] = ( ok ? QString::number( expires ) : QString::number( 0 ) );
+
+        /**
+         * If the item has expired, cache it for one hour and try and refetch later
+         */
+        emit updateCache( criteria, (maxAge == 0 ? 3600000 /* One hour */ : maxAge), requestData.type, returnedData );
     }
     else
         tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Network error in fetching newrelease:" << reply->url().toString();
 }
 
-
 Q_EXPORT_PLUGIN2( Tomahawk::InfoSystem::InfoPlugin, Tomahawk::InfoSystem::NewReleasesPlugin )

src/infoplugins/generic/newreleases/NewReleasesPlugin.h

@@ -1,5 +1,6 @@
 /* === This file is part of Tomahawk Player - <http://tomahawk-player.org> ===
  *
+ *   Copyright 2012, Hugo Lindström <hugolm84@gmail.com>
  *   Copyright 2012, Casey Link <unnamedrambler@gmail.com>
  *
  *   Tomahawk is free software: you can redistribute it and/or modify
@@ -22,7 +23,7 @@
 #include "infosystem/InfoSystem.h"
 #include "infosystem/InfoSystemWorker.h"
 #include "infoplugins/InfoPluginDllMacro.h"
-
+#include <QVariantMap>
 #include <QtNetwork/QNetworkReply>
 #include <QtCore/QObject>
 
@@ -38,7 +39,6 @@ class INFOPLUGINDLLEXPORT NewReleasesPlugin : public InfoPlugin
 {
     Q_OBJECT
     Q_INTERFACES( Tomahawk::InfoSystem::InfoPlugin )
-
 public:
     NewReleasesPlugin();
     virtual ~NewReleasesPlugin();
@@ -69,11 +69,6 @@ protected slots:
     void nrReturned();
 
 private:
-    /**
-     * Fetch list of newlreeases sources (e.g., rovi)
-     * Populates the m_nrSources member.
-     */
-    void fetchNRSourcesList( bool fetchOnlySourcesList );
     /**
      * Requests newrelease list for each source in m_chartSources
      */
@@ -87,10 +82,12 @@ private:
     void fetchNRCapabilitiesFromCache( Tomahawk::InfoSystem::InfoRequestData requestData );
     void dataError( Tomahawk::InfoSystem::InfoRequestData requestData );
 
-    QStringList m_nrSources;
+    qlonglong getMaxAge( const QByteArray &rawHeader ) const;
+    qlonglong getMaxAge( const qlonglong expires ) const;
+
+    QList< Tomahawk::InfoSystem::InfoStringHash > m_nrSources;
+    QStringList m_refetchSource;
     QString m_nrVersion;
-    QList< InfoStringHash > m_newreleases;
-    //ChartType m_chartType;
     QVariantMap m_allNRsMap;
     uint m_nrFetchJobs;
     QList< InfoRequestData > m_cachedRequests;

src/libtomahawk/widgets/Breadcrumb.h

@@ -46,7 +46,8 @@ public:
     enum ExtraRoles {
         DefaultRole = Qt::UserRole + 1,
         UserSelectedRole = Qt::UserRole + 2,
-        ChartIdRole = Qt::UserRole + 3
+        ChartIdRole = Qt::UserRole + 3,
+        ChartExpireRole = Qt::UserRole + 4
     };
 
     explicit Breadcrumb( QWidget* parent = 0, Qt::WindowFlags f = 0 );

src/libtomahawk/widgets/NewReleasesWidget.cpp

@@ -175,7 +175,7 @@ NewReleasesWidget::infoSystemInfo( Tomahawk::InfoSystem::InfoRequestData request
             const QString type = returnedData["type"].toString();
             if( !returnedData.contains(type) )
                 break;
-            const QString side = requestData.customData["whatshot_side"].toString();
+
             const QString releaseId = requestData.input.value< Tomahawk::InfoSystem::InfoStringHash >().value( "nr_id" );
 
             m_queuedFetches.remove( releaseId );
@@ -241,6 +241,7 @@ NewReleasesWidget::leftCrumbIndexChanged( QModelIndex index )
 
 
     const QString nrId = item->data( Breadcrumb::ChartIdRole ).toString();
+    const qlonglong nrExpires = item->data( Breadcrumb::ChartExpireRole ).toLongLong();
 
     if ( m_albumModels.contains( nrId ) )
     {
@@ -255,6 +256,7 @@ NewReleasesWidget::leftCrumbIndexChanged( QModelIndex index )
 
     Tomahawk::InfoSystem::InfoStringHash criteria;
     criteria.insert( "nr_id", nrId );
+    criteria.insert( "nr_expires", QString::number(nrExpires) );
     /// Remember to lower the source!
     criteria.insert( "nr_source",  index.data().toString().toLower() );
 
@@ -307,6 +309,7 @@ NewReleasesWidget::parseNode( QStandardItem* parentItem, const QString &label, c
         {
             QStandardItem *childItem= new QStandardItem( chart[ "label" ] );
             childItem->setData( chart[ "id" ], Breadcrumb::ChartIdRole );
+            childItem->setData( chart[ "expires" ], Breadcrumb::ChartExpireRole );
             if ( chart.value( "default", "" ) == "true")
             {
                 childItem->setData( true, Breadcrumb::DefaultRole );