diff --git a/src/infoplugins/generic/charts/ChartsPlugin.cpp b/src/infoplugins/generic/charts/ChartsPlugin.cpp index 97af96217..82f9e0390 100644 --- a/src/infoplugins/generic/charts/ChartsPlugin.cpp +++ b/src/infoplugins/generic/charts/ChartsPlugin.cpp @@ -68,17 +68,29 @@ ChartsPlugin::~ChartsPlugin() void ChartsPlugin::init() { - tDebug( LOGVERBOSE ) << Q_FUNC_INFO << QThread::currentThread(); - QVariantList source_qvarlist = TomahawkUtils::Cache::instance()->getData( "ChartsPlugin", "chart_sources" ).toList(); - - foreach ( const QVariant & source, source_qvarlist ) + QVariant data = TomahawkUtils::Cache::instance()->getData( "ChartsPlugin", "chart_sources" ); + if ( data.canConvert< QList< Tomahawk::InfoSystem::InfoStringHash > >() ) { - m_chartResources.append( source.toString() ); - tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "fetched source from cache" << source.toString(); - + const QList< Tomahawk::InfoSystem::InfoStringHash > sourceList = data.value< QList< Tomahawk::InfoSystem::InfoStringHash > >(); + foreach ( const Tomahawk::InfoSystem::InfoStringHash &sourceHash, sourceList ) + { + bool ok; + qlonglong maxAge = getMaxAge( QString( sourceHash[ "chart_expires" ] ).toLongLong( &ok ) ); + if ( !ok || maxAge <= 0 ) + { + // This source has expired. + m_refetchSource << sourceHash[ "chart_source" ]; + } + m_chartResources << sourceHash; + } } - tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "total sources" << m_chartResources.size() << source_qvarlist.size(); + tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "total sources" << m_chartResources.size() << m_chartResources; + + if( m_chartResources.size() == 0 || m_refetchSource.size() != 0 ) + { + tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Will refetch on next request. Empty or Invalid CACHE" << m_chartResources.size() << m_refetchSource; + } } @@ -109,11 +121,10 @@ ChartsPlugin::getInfo( Tomahawk::InfoSystem::InfoRequestData requestData ) } else { - foreach ( QString resource, m_chartResources ) + foreach ( const Tomahawk::InfoSystem::InfoStringHash &sourceHash, m_chartResources ) { - if ( resource == hash["chart_source"] ) + if ( sourceHash[ "chart_source" ] == hash[ "chart_source" ] ) { - tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "get source" << resource; foundSource = true; } } @@ -151,8 +162,8 @@ ChartsPlugin::fetchChartFromCache( Tomahawk::InfoSystem::InfoRequestData request InfoStringHash hash = requestData.input.value< Tomahawk::InfoSystem::InfoStringHash >(); Tomahawk::InfoSystem::InfoStringHash criteria; - /// Each request needs to contain both a id and source - if ( !hash.contains( "chart_id" ) && !hash.contains( "chart_source" ) ) + /// Each request needs to contain both a id, source and expires param + if ( !hash.contains( "chart_id" ) && !hash.contains( "chart_source" ) && !hash.contains( "chart_expires" ) ) { tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Hash did not contain required params!"; dataError( requestData ); @@ -160,11 +171,22 @@ ChartsPlugin::fetchChartFromCache( Tomahawk::InfoSystem::InfoRequestData request } /// Set the criterias for current chart - criteria["chart_id"] = hash["chart_id"]; - criteria["chart_source"] = hash["chart_source"]; + criteria[ "chart_id" ] = hash[ "chart_id" ]; + criteria[ "chart_source" ] = hash[ "chart_source" ]; + criteria[ "chart_expires" ] = hash[ "chart_expires" ]; tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Checking cache for " << hash["chart_id"] << " from " << hash["chart_source"]; - emit getCachedInfo( criteria, 86400000, requestData ); + bool ok; + const qlonglong maxAge = getMaxAge( QString( hash[ "chart_expires" ] ).toLongLong( &ok ) ); + + if ( !ok || maxAge <= 0 ) + { + emit notInCacheSlot( criteria, requestData ); + return; + } + + emit getCachedInfo( criteria, maxAge, requestData ); + return; } @@ -181,8 +203,28 @@ ChartsPlugin::fetchChartCapabilitiesFromCache( Tomahawk::InfoSystem::InfoRequest Tomahawk::InfoSystem::InfoStringHash criteria; criteria[ "InfoChartCapabilities" ] = "chartsplugin"; criteria[ "InfoChartVersion" ] = m_chartVersion; + + Tomahawk::InfoSystem::InfoStringHash inputData = requestData.input.value< Tomahawk::InfoSystem::InfoStringHash >(); + + /// @todo: Only fetch this source, and update charts map + if( inputData.contains( "chart_refetch" ) ) + { + tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Chart source needs to be refetched!" << inputData[ "chart_refetch" ]; + m_refetchSource << inputData[ "chart_refetch" ]; + } + + /// Someone requested capabilities, but init() told us someone was out of date + /// Next fetch will fetch those that are invalid + + if ( m_refetchSource.size() != 0 ) + { + tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Some sources need to refetch!" << m_refetchSource; + emit notInCacheSlot( criteria, requestData ); + return; + } + tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Checking cache for " << "InfoChartCapabilities" << m_chartVersion; - emit getCachedInfo( criteria, 864000000, requestData ); + emit getCachedInfo ( criteria, 172800000 /* 2 days */, requestData ); } @@ -219,7 +261,9 @@ ChartsPlugin::notInCacheSlot( QHash criteria, Tomahawk::InfoSy void ChartsPlugin::fetchChartSourcesList( bool fetchOnlySourceList ) { - QUrl url = QUrl( QString( CHART_URL "charts" ) ); + QUrl url = QUrl( QString ( CHART_URL "charts" ) ); + url.addQueryItem( "version", TomahawkUtils::appFriendlyVersion() ); + QNetworkReply* reply = TomahawkUtils::nam()->get( QNetworkRequest( url ) ); reply->setProperty( "only_source_list", fetchOnlySourceList ); @@ -246,33 +290,73 @@ ChartsPlugin::chartSourcesList() return; } + /// This will re/fetch all sources + /// @todo: Fetch only requested source m_chartResources.clear(); - foreach ( const QVariant& source, sources ) + + foreach ( const QVariant &rsource, sources ) { - m_chartResources << source.toString(); + /// Each item has an expiration, on next request for cache, it will be checked */ + const QString source = rsource.toString(); + + /// Twisted backend Uppers first header letter, and lowers the rest + QString tmpSource = source + "expires"; + tmpSource[0] = tmpSource[0].toUpper(); + + const QString headerExpiration = reply->rawHeader( QString( tmpSource ).toLocal8Bit() ); + const qlonglong maxAge = getMaxAge( headerExpiration.toLocal8Bit() ); + const qlonglong expires = headerExpiration.toLongLong(&ok); + Tomahawk::InfoSystem::InfoStringHash source_expire; + + if ( ok ) + { + source_expire[ "chart_source" ] = source; + source_expire[ "chart_expires" ] = QString::number(expires); + m_chartResources << source_expire; + } + + if ( maxAge == 0 ) + { + tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "MaxAge for " << source << " is 0. Fetching all"; + reply->setProperty( "only_source_list", false ); + } + } + + /// We can store the source list for how long as we want + /// In init, we check expiration for each source, and refetch if invalid + /// 2 days seems fair enough though tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "storing sources in cache" << m_chartResources; - TomahawkUtils::Cache::instance()->putData( "ChartsPlugin", 172800000 /* 2 days */, "chart_sources", m_chartResources ); - if ( !reply->property( "only_source_list" ).toBool() ) + TomahawkUtils::Cache::instance()->putData( "ChartsPlugin", 172800000 /* 2 days */, "chart_sources", QVariant::fromValue< QList< Tomahawk::InfoSystem::InfoStringHash > > ( m_chartResources ) ); + + if( !reply->property( "only_source_list" ).toBool() ) + { + tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Fetching all sources!"; fetchAllChartSources(); + } } else tDebug() << Q_FUNC_INFO << "Encountered error fetching chart sources list"; } + void ChartsPlugin::fetchAllChartSources() { if ( !m_chartResources.isEmpty() && m_allChartsMap.isEmpty() ) { - tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "InfoChart fetching source data"; - foreach ( QString source, m_chartResources ) + tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "InfoNewRelease fetching source data"; + foreach ( const Tomahawk::InfoSystem::InfoStringHash source, m_chartResources ) { - QUrl url = QUrl( QString( CHART_URL "charts/%1" ).arg( source ) ); + QUrl url = QUrl( QString( CHART_URL "charts/%1" ).arg( source[ "chart_source" ] ) ); + url.addQueryItem( "version", TomahawkUtils::appFriendlyVersion() ); + QNetworkReply* reply = TomahawkUtils::nam()->get( QNetworkRequest( url ) ); - reply->setProperty( "chart_source", source ); + reply->setProperty( "chart_source", source[ "chart_source" ] ); + + tDebug() << Q_FUNC_INFO << "fetching:" << url; connect( reply, SIGNAL( finished() ), SLOT( chartsList() ) ); m_chartsFetchJobs++; @@ -285,13 +369,15 @@ void ChartsPlugin::fetchChart( Tomahawk::InfoSystem::InfoRequestData requestData, const QString& source, const QString& chart_id ) { /// Fetch the chart, we need source and id - QUrl url = QUrl( QString( CHART_URL "charts/%1/%2" ).arg( source ).arg( chart_id ) ); - tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "fetching:" << url; + QUrl url = QUrl ( QString ( CHART_URL "charts/%1/%2" ).arg( source ).arg( chart_id ) ); + url.addQueryItem( "version", TomahawkUtils::appFriendlyVersion() ); + + tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "fetching: " << url; QNetworkReply* reply = TomahawkUtils::nam()->get( QNetworkRequest( url ) ); reply->setProperty( "requestData", QVariant::fromValue< Tomahawk::InfoSystem::InfoRequestData >( requestData ) ); - connect( reply, SIGNAL( finished() ), SLOT( chartReturned() ) ); + connect ( reply, SIGNAL( finished() ), SLOT( chartReturned() ) ); } @@ -316,7 +402,13 @@ ChartsPlugin::chartsList() /// Got types, append! const QString source = reply->property("chart_source").toString(); + const qlonglong expires = QString( reply->rawHeader( QString( "Expires" ).toLocal8Bit() ) ).toLongLong( &ok ); + if( !ok ) + { + tDebug( LOGVERBOSE ) << Q_FUNC_INFO << reply->url(); + tDebug( LOGVERBOSE ) << Q_FUNC_INFO << reply->rawHeaderPairs(); + } // We'll populate charts with the data from the server QVariantMap charts; QString chartName; @@ -341,7 +433,7 @@ ChartsPlugin::chartsList() const QString geo = chart.value( "geo" ).toString(); QString name = chart.value( "genre" ).toString(); const QString type = QString( chart.value( "type" ).toString() + "s" ); - const bool isDefault = ( chart.contains( "default" ) && chart[ "default" ].toInt() == 1 ); + bool isDefault = ( chart.contains( "default" ) && chart[ "default" ].toInt() == 1 ); // Hack! // Japan charts contains multiple duplicates, all which are linked @@ -359,7 +451,6 @@ ChartsPlugin::chartsList() { if ( !m_cachedCountries.contains( geo ) ) { - QLocale l( QString( "en_%1" ).arg( geo ) ); extra = Tomahawk::CountryUtils::fullCountryFromCode( geo ); for ( int i = 1; i < extra.size(); i++ ) @@ -386,7 +477,12 @@ ChartsPlugin::chartsList() c[ "label" ] = name; c[ "type" ] = "album"; if ( isDefault ) + { c[ "default" ] = "true"; + } + + /// If this item has expired, set it to 0. + c[ "expires" ] = ( ok ? QString::number (expires ) : QString::number( 0 ) ); QList< Tomahawk::InfoSystem::InfoStringHash > extraTypeData = extraType[ extra ][ type ].value< QList< Tomahawk::InfoSystem::InfoStringHash > >(); extraTypeData.append( c ); @@ -403,7 +499,6 @@ ChartsPlugin::chartsList() foreach ( const QString& c, extraType.keys() ) { charts[ c ] = extraType[ c ]; -// tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "extraType has types:" << c; } if ( source == "itunes" ) { @@ -438,7 +533,7 @@ ChartsPlugin::chartsList() InfoStringHash c; c[ "id" ] = chart.value( "id" ).toString(); - + c[ "expires" ] = ( ok ? QString::number( expires ) : QString::number( 0 ) ); if ( chart.value( "genre").isValid() ) c[ "label" ] = chart.value( "genre" ).toString(); else @@ -509,7 +604,9 @@ ChartsPlugin::chartsList() Tomahawk::InfoSystem::InfoStringHash criteria; criteria[ "InfoChartCapabilities" ] = "chartsplugin"; criteria[ "InfoChartVersion" ] = m_chartVersion; - emit updateCache( criteria, 864000000, request.type, m_allChartsMap ); + + /// We can cache it the lot for 2 days, it will be checked on next request + emit updateCache( criteria, 172800000 /* 2 days */, request.type,m_allChartsMap ); } m_cachedRequests.clear(); } @@ -535,6 +632,9 @@ ChartsPlugin::chartReturned() return; } + const qlonglong maxAge = getMaxAge( reply->rawHeader( QString( "Expires" ).toLocal8Bit() ) ); + const qlonglong expires = QString( reply->rawHeader( QString( "Expires" ).toLocal8Bit() ) ).toLongLong( &ok ); + /// SO we have a result, parse it! QVariantList chartResponse = res.value( "list" ).toList(); QList< Tomahawk::InfoSystem::InfoStringHash > top_tracks; @@ -637,10 +737,47 @@ ChartsPlugin::chartReturned() Tomahawk::InfoSystem::InfoStringHash origData = requestData.input.value< Tomahawk::InfoSystem::InfoStringHash >(); criteria[ "chart_id" ] = origData[ "chart_id" ]; criteria[ "chart_source" ] = origData[ "chart_source" ]; - emit updateCache( criteria, 86400000, requestData.type, returnedData ); + criteria[ "chart_expires" ] = ( ok ? QString::number( expires ) : QString::number( 0 ) ); + + /// If the item has expired, cache it for one hour and try and refetch later + emit updateCache( criteria, (maxAge == 0 ? 3600000 /* One hour */ : maxAge), requestData.type, returnedData ); } else + { tDebug() << Q_FUNC_INFO << "Network error in fetching chart:" << reply->url().toString(); + Tomahawk::InfoSystem::InfoRequestData requestData = reply->property( "requestData" ).value< Tomahawk::InfoSystem::InfoRequestData >(); + Tomahawk::InfoSystem::InfoStringHash origData = requestData.input.value< Tomahawk::InfoSystem::InfoStringHash >(); + returnedData[ "chart_error" ] = "Invalid ID"; + returnedData[ "chart_source" ] = origData[ "chart_source" ]; + returnedData[ "chart_id" ] = origData[ "chart_id" ]; + emit info( requestData, returnedData ); + } +} + +qlonglong +ChartsPlugin::getMaxAge( const QByteArray &rawHeader ) const +{ + bool ok; + qlonglong expires = QString( rawHeader ).toLongLong( &ok ); + if ( ok ) + { + tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Got rawheader " << QString( rawHeader ) << ":" << expires; + return getMaxAge( expires ); + } + return 0; +} + +qlonglong +ChartsPlugin::getMaxAge( const qlonglong expires ) const +{ + qlonglong currentEpoch = QDateTime::currentMSecsSinceEpoch()/1000; + qlonglong expiresInSeconds = expires-currentEpoch; + + if ( expiresInSeconds > 0 ) + { + return ( qlonglong )expiresInSeconds*1000; + } + return 0; } } diff --git a/src/infoplugins/generic/charts/ChartsPlugin.h b/src/infoplugins/generic/charts/ChartsPlugin.h index bc0fbacfe..07003a1d9 100644 --- a/src/infoplugins/generic/charts/ChartsPlugin.h +++ b/src/infoplugins/generic/charts/ChartsPlugin.h @@ -99,9 +99,13 @@ private: void fetchChartCapabilitiesFromCache( Tomahawk::InfoSystem::InfoRequestData requestData ); void dataError( Tomahawk::InfoSystem::InfoRequestData requestData ); - QStringList m_chartResources; + + qlonglong getMaxAge( const QByteArray &rawHeader ) const; + qlonglong getMaxAge( const qlonglong expires ) const; + + QList< Tomahawk::InfoSystem::InfoStringHash > m_chartResources; + QStringList m_refetchSource; QString m_chartVersion; - QList< InfoStringHash > m_charts; ChartType m_chartType; QVariantMap m_allChartsMap; uint m_chartsFetchJobs; diff --git a/src/libtomahawk/widgets/WhatsHotWidget.cpp b/src/libtomahawk/widgets/WhatsHotWidget.cpp index 21de28e34..72597715b 100644 --- a/src/libtomahawk/widgets/WhatsHotWidget.cpp +++ b/src/libtomahawk/widgets/WhatsHotWidget.cpp @@ -175,12 +175,12 @@ void WhatsHotWidget::fetchData() { - Tomahawk::InfoSystem::InfoStringHash artistInfo; + Tomahawk::InfoSystem::InfoStringHash criteria; Tomahawk::InfoSystem::InfoRequestData requestData; requestData.caller = s_whatsHotIdentifier; requestData.customData = QVariantMap(); - requestData.input = QVariant::fromValue< Tomahawk::InfoSystem::InfoStringHash >( artistInfo ); + requestData.input = QVariant::fromValue< Tomahawk::InfoSystem::InfoStringHash >( criteria ); requestData.type = Tomahawk::InfoSystem::InfoChartCapabilities; requestData.timeoutMillis = 20000; requestData.allSources = true; @@ -277,9 +277,32 @@ WhatsHotWidget::infoSystemInfo( Tomahawk::InfoSystem::InfoRequestData requestDat case InfoSystem::InfoChart: { + + if( returnedData.contains( "chart_error") ) + { + tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Info came back with error!!"; + + Tomahawk::InfoSystem::InfoStringHash criteria; + criteria.insert( "chart_refetch", returnedData[ "chart_source" ].value< QString >() ); + + Tomahawk::InfoSystem::InfoRequestData requestData; + requestData.caller = s_whatsHotIdentifier; + requestData.customData = QVariantMap(); + requestData.input = QVariant::fromValue< Tomahawk::InfoSystem::InfoStringHash >( criteria ); + requestData.type = Tomahawk::InfoSystem::InfoChartCapabilities; + requestData.timeoutMillis = 20000; + requestData.allSources = false; + Tomahawk::InfoSystem::InfoSystem::instance()->getInfo( requestData ); + + tDebug( LOGVERBOSE ) << "WhatsHot: re-requesting InfoChartCapabilities"; + break; + } + if ( !returnedData.contains( "type" ) ) break; + const QString type = returnedData[ "type" ].toString(); + if ( !returnedData.contains( type ) ) break; @@ -362,7 +385,6 @@ WhatsHotWidget::leftCrumbIndexChanged( QModelIndex index ) { tDebug( LOGVERBOSE ) << "WhatsHot: left crumb changed" << index.data(); - QStandardItem* item = m_crumbModelLeft->itemFromIndex( m_sortedProxy->mapToSource( index ) ); if ( !item ) return; @@ -382,6 +404,7 @@ WhatsHotWidget::leftCrumbIndexChanged( QModelIndex index ) curr.prepend( index.data().toString().toLower() ); } const QString chartId = item->data( Breadcrumb::ChartIdRole ).toString(); + const qlonglong chartExpires = item->data( Breadcrumb::ChartExpireRole ).toLongLong(); const QString chartSource = curr.takeFirst().toLower(); curr.append( chartSource ); curr.append( chartId ); @@ -413,9 +436,9 @@ WhatsHotWidget::leftCrumbIndexChanged( QModelIndex index ) Tomahawk::InfoSystem::InfoStringHash criteria; criteria.insert( "chart_id", chartId ); + criteria.insert( "chart_expires", QString::number( chartExpires ) ); /// Remember to lower the source! criteria.insert( "chart_source", index.data().toString().toLower() ); - Tomahawk::InfoSystem::InfoRequestData requestData; QVariantMap customData; customData.insert( "whatshot_side", "left" ); @@ -466,6 +489,7 @@ WhatsHotWidget::parseNode( QStandardItem* parentItem, const QString &label, cons { QStandardItem *childItem= new QStandardItem( chart[ "label" ] ); childItem->setData( chart[ "id" ], Breadcrumb::ChartIdRole ); + childItem->setData( chart[ "expires" ], Breadcrumb::ChartExpireRole ); if ( m_currentVIds.contains( chart.value( "id" ).toLower() ) ) {