1
0
mirror of https://github.com/tomahawk-player/tomahawk.git synced 2025-03-23 01:09:42 +01:00

Better cachevalidation. Refetch sources if fail to get id

This commit is contained in:
Hugo Lindström 2012-11-28 20:15:35 +01:00
parent 53251c1ba5
commit 42dbdd82c9
3 changed files with 206 additions and 41 deletions

View File

@ -68,17 +68,29 @@ ChartsPlugin::~ChartsPlugin()
void
ChartsPlugin::init()
{
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << QThread::currentThread();
QVariantList source_qvarlist = TomahawkUtils::Cache::instance()->getData( "ChartsPlugin", "chart_sources" ).toList();
foreach ( const QVariant & source, source_qvarlist )
QVariant data = TomahawkUtils::Cache::instance()->getData( "ChartsPlugin", "chart_sources" );
if ( data.canConvert< QList< Tomahawk::InfoSystem::InfoStringHash > >() )
{
m_chartResources.append( source.toString() );
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "fetched source from cache" << source.toString();
const QList< Tomahawk::InfoSystem::InfoStringHash > sourceList = data.value< QList< Tomahawk::InfoSystem::InfoStringHash > >();
foreach ( const Tomahawk::InfoSystem::InfoStringHash &sourceHash, sourceList )
{
bool ok;
qlonglong maxAge = getMaxAge( QString( sourceHash[ "chart_expires" ] ).toLongLong( &ok ) );
if ( !ok || maxAge <= 0 )
{
// This source has expired.
m_refetchSource << sourceHash[ "chart_source" ];
}
m_chartResources << sourceHash;
}
}
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "total sources" << m_chartResources.size() << source_qvarlist.size();
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "total sources" << m_chartResources.size() << m_chartResources;
if( m_chartResources.size() == 0 || m_refetchSource.size() != 0 )
{
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Will refetch on next request. Empty or Invalid CACHE" << m_chartResources.size() << m_refetchSource;
}
}
@ -109,11 +121,10 @@ ChartsPlugin::getInfo( Tomahawk::InfoSystem::InfoRequestData requestData )
}
else
{
foreach ( QString resource, m_chartResources )
foreach ( const Tomahawk::InfoSystem::InfoStringHash &sourceHash, m_chartResources )
{
if ( resource == hash["chart_source"] )
if ( sourceHash[ "chart_source" ] == hash[ "chart_source" ] )
{
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "get source" << resource;
foundSource = true;
}
}
@ -151,8 +162,8 @@ ChartsPlugin::fetchChartFromCache( Tomahawk::InfoSystem::InfoRequestData request
InfoStringHash hash = requestData.input.value< Tomahawk::InfoSystem::InfoStringHash >();
Tomahawk::InfoSystem::InfoStringHash criteria;
/// Each request needs to contain both a id and source
if ( !hash.contains( "chart_id" ) && !hash.contains( "chart_source" ) )
/// Each request needs to contain both a id, source and expires param
if ( !hash.contains( "chart_id" ) && !hash.contains( "chart_source" ) && !hash.contains( "chart_expires" ) )
{
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Hash did not contain required params!";
dataError( requestData );
@ -160,11 +171,22 @@ ChartsPlugin::fetchChartFromCache( Tomahawk::InfoSystem::InfoRequestData request
}
/// Set the criterias for current chart
criteria["chart_id"] = hash["chart_id"];
criteria["chart_source"] = hash["chart_source"];
criteria[ "chart_id" ] = hash[ "chart_id" ];
criteria[ "chart_source" ] = hash[ "chart_source" ];
criteria[ "chart_expires" ] = hash[ "chart_expires" ];
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Checking cache for " << hash["chart_id"] << " from " << hash["chart_source"];
emit getCachedInfo( criteria, 86400000, requestData );
bool ok;
const qlonglong maxAge = getMaxAge( QString( hash[ "chart_expires" ] ).toLongLong( &ok ) );
if ( !ok || maxAge <= 0 )
{
emit notInCacheSlot( criteria, requestData );
return;
}
emit getCachedInfo( criteria, maxAge, requestData );
return;
}
@ -181,8 +203,28 @@ ChartsPlugin::fetchChartCapabilitiesFromCache( Tomahawk::InfoSystem::InfoRequest
Tomahawk::InfoSystem::InfoStringHash criteria;
criteria[ "InfoChartCapabilities" ] = "chartsplugin";
criteria[ "InfoChartVersion" ] = m_chartVersion;
Tomahawk::InfoSystem::InfoStringHash inputData = requestData.input.value< Tomahawk::InfoSystem::InfoStringHash >();
/// @todo: Only fetch this source, and update charts map
if( inputData.contains( "chart_refetch" ) )
{
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Chart source needs to be refetched!" << inputData[ "chart_refetch" ];
m_refetchSource << inputData[ "chart_refetch" ];
}
/// Someone requested capabilities, but init() told us someone was out of date
/// Next fetch will fetch those that are invalid
if ( m_refetchSource.size() != 0 )
{
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Some sources need to refetch!" << m_refetchSource;
emit notInCacheSlot( criteria, requestData );
return;
}
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Checking cache for " << "InfoChartCapabilities" << m_chartVersion;
emit getCachedInfo( criteria, 864000000, requestData );
emit getCachedInfo ( criteria, 172800000 /* 2 days */, requestData );
}
@ -219,7 +261,9 @@ ChartsPlugin::notInCacheSlot( QHash<QString, QString> criteria, Tomahawk::InfoSy
void
ChartsPlugin::fetchChartSourcesList( bool fetchOnlySourceList )
{
QUrl url = QUrl( QString( CHART_URL "charts" ) );
QUrl url = QUrl( QString ( CHART_URL "charts" ) );
url.addQueryItem( "version", TomahawkUtils::appFriendlyVersion() );
QNetworkReply* reply = TomahawkUtils::nam()->get( QNetworkRequest( url ) );
reply->setProperty( "only_source_list", fetchOnlySourceList );
@ -246,33 +290,73 @@ ChartsPlugin::chartSourcesList()
return;
}
/// This will re/fetch all sources
/// @todo: Fetch only requested source
m_chartResources.clear();
foreach ( const QVariant& source, sources )
foreach ( const QVariant &rsource, sources )
{
m_chartResources << source.toString();
/// Each item has an expiration, on next request for cache, it will be checked */
const QString source = rsource.toString();
/// Twisted backend Uppers first header letter, and lowers the rest
QString tmpSource = source + "expires";
tmpSource[0] = tmpSource[0].toUpper();
const QString headerExpiration = reply->rawHeader( QString( tmpSource ).toLocal8Bit() );
const qlonglong maxAge = getMaxAge( headerExpiration.toLocal8Bit() );
const qlonglong expires = headerExpiration.toLongLong(&ok);
Tomahawk::InfoSystem::InfoStringHash source_expire;
if ( ok )
{
source_expire[ "chart_source" ] = source;
source_expire[ "chart_expires" ] = QString::number(expires);
m_chartResources << source_expire;
}
if ( maxAge == 0 )
{
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "MaxAge for " << source << " is 0. Fetching all";
reply->setProperty( "only_source_list", false );
}
}
/// We can store the source list for how long as we want
/// In init, we check expiration for each source, and refetch if invalid
/// 2 days seems fair enough though
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "storing sources in cache" << m_chartResources;
TomahawkUtils::Cache::instance()->putData( "ChartsPlugin", 172800000 /* 2 days */, "chart_sources", m_chartResources );
if ( !reply->property( "only_source_list" ).toBool() )
TomahawkUtils::Cache::instance()->putData( "ChartsPlugin", 172800000 /* 2 days */, "chart_sources", QVariant::fromValue< QList< Tomahawk::InfoSystem::InfoStringHash > > ( m_chartResources ) );
if( !reply->property( "only_source_list" ).toBool() )
{
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Fetching all sources!";
fetchAllChartSources();
}
}
else
tDebug() << Q_FUNC_INFO << "Encountered error fetching chart sources list";
}
void
ChartsPlugin::fetchAllChartSources()
{
if ( !m_chartResources.isEmpty() && m_allChartsMap.isEmpty() )
{
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "InfoChart fetching source data";
foreach ( QString source, m_chartResources )
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "InfoNewRelease fetching source data";
foreach ( const Tomahawk::InfoSystem::InfoStringHash source, m_chartResources )
{
QUrl url = QUrl( QString( CHART_URL "charts/%1" ).arg( source ) );
QUrl url = QUrl( QString( CHART_URL "charts/%1" ).arg( source[ "chart_source" ] ) );
url.addQueryItem( "version", TomahawkUtils::appFriendlyVersion() );
QNetworkReply* reply = TomahawkUtils::nam()->get( QNetworkRequest( url ) );
reply->setProperty( "chart_source", source );
reply->setProperty( "chart_source", source[ "chart_source" ] );
tDebug() << Q_FUNC_INFO << "fetching:" << url;
connect( reply, SIGNAL( finished() ), SLOT( chartsList() ) );
m_chartsFetchJobs++;
@ -285,13 +369,15 @@ void
ChartsPlugin::fetchChart( Tomahawk::InfoSystem::InfoRequestData requestData, const QString& source, const QString& chart_id )
{
/// Fetch the chart, we need source and id
QUrl url = QUrl( QString( CHART_URL "charts/%1/%2" ).arg( source ).arg( chart_id ) );
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "fetching:" << url;
QUrl url = QUrl ( QString ( CHART_URL "charts/%1/%2" ).arg( source ).arg( chart_id ) );
url.addQueryItem( "version", TomahawkUtils::appFriendlyVersion() );
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "fetching: " << url;
QNetworkReply* reply = TomahawkUtils::nam()->get( QNetworkRequest( url ) );
reply->setProperty( "requestData", QVariant::fromValue< Tomahawk::InfoSystem::InfoRequestData >( requestData ) );
connect( reply, SIGNAL( finished() ), SLOT( chartReturned() ) );
connect ( reply, SIGNAL( finished() ), SLOT( chartReturned() ) );
}
@ -316,7 +402,13 @@ ChartsPlugin::chartsList()
/// Got types, append!
const QString source = reply->property("chart_source").toString();
const qlonglong expires = QString( reply->rawHeader( QString( "Expires" ).toLocal8Bit() ) ).toLongLong( &ok );
if( !ok )
{
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << reply->url();
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << reply->rawHeaderPairs();
}
// We'll populate charts with the data from the server
QVariantMap charts;
QString chartName;
@ -341,7 +433,7 @@ ChartsPlugin::chartsList()
const QString geo = chart.value( "geo" ).toString();
QString name = chart.value( "genre" ).toString();
const QString type = QString( chart.value( "type" ).toString() + "s" );
const bool isDefault = ( chart.contains( "default" ) && chart[ "default" ].toInt() == 1 );
bool isDefault = ( chart.contains( "default" ) && chart[ "default" ].toInt() == 1 );
// Hack!
// Japan charts contains multiple duplicates, all which are linked
@ -359,7 +451,6 @@ ChartsPlugin::chartsList()
{
if ( !m_cachedCountries.contains( geo ) )
{
QLocale l( QString( "en_%1" ).arg( geo ) );
extra = Tomahawk::CountryUtils::fullCountryFromCode( geo );
for ( int i = 1; i < extra.size(); i++ )
@ -386,7 +477,12 @@ ChartsPlugin::chartsList()
c[ "label" ] = name;
c[ "type" ] = "album";
if ( isDefault )
{
c[ "default" ] = "true";
}
/// If this item has expired, set it to 0.
c[ "expires" ] = ( ok ? QString::number (expires ) : QString::number( 0 ) );
QList< Tomahawk::InfoSystem::InfoStringHash > extraTypeData = extraType[ extra ][ type ].value< QList< Tomahawk::InfoSystem::InfoStringHash > >();
extraTypeData.append( c );
@ -403,7 +499,6 @@ ChartsPlugin::chartsList()
foreach ( const QString& c, extraType.keys() )
{
charts[ c ] = extraType[ c ];
// tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "extraType has types:" << c;
}
if ( source == "itunes" )
{
@ -438,7 +533,7 @@ ChartsPlugin::chartsList()
InfoStringHash c;
c[ "id" ] = chart.value( "id" ).toString();
c[ "expires" ] = ( ok ? QString::number( expires ) : QString::number( 0 ) );
if ( chart.value( "genre").isValid() )
c[ "label" ] = chart.value( "genre" ).toString();
else
@ -509,7 +604,9 @@ ChartsPlugin::chartsList()
Tomahawk::InfoSystem::InfoStringHash criteria;
criteria[ "InfoChartCapabilities" ] = "chartsplugin";
criteria[ "InfoChartVersion" ] = m_chartVersion;
emit updateCache( criteria, 864000000, request.type, m_allChartsMap );
/// We can cache it the lot for 2 days, it will be checked on next request
emit updateCache( criteria, 172800000 /* 2 days */, request.type,m_allChartsMap );
}
m_cachedRequests.clear();
}
@ -535,6 +632,9 @@ ChartsPlugin::chartReturned()
return;
}
const qlonglong maxAge = getMaxAge( reply->rawHeader( QString( "Expires" ).toLocal8Bit() ) );
const qlonglong expires = QString( reply->rawHeader( QString( "Expires" ).toLocal8Bit() ) ).toLongLong( &ok );
/// SO we have a result, parse it!
QVariantList chartResponse = res.value( "list" ).toList();
QList< Tomahawk::InfoSystem::InfoStringHash > top_tracks;
@ -637,10 +737,47 @@ ChartsPlugin::chartReturned()
Tomahawk::InfoSystem::InfoStringHash origData = requestData.input.value< Tomahawk::InfoSystem::InfoStringHash >();
criteria[ "chart_id" ] = origData[ "chart_id" ];
criteria[ "chart_source" ] = origData[ "chart_source" ];
emit updateCache( criteria, 86400000, requestData.type, returnedData );
criteria[ "chart_expires" ] = ( ok ? QString::number( expires ) : QString::number( 0 ) );
/// If the item has expired, cache it for one hour and try and refetch later
emit updateCache( criteria, (maxAge == 0 ? 3600000 /* One hour */ : maxAge), requestData.type, returnedData );
}
else
{
tDebug() << Q_FUNC_INFO << "Network error in fetching chart:" << reply->url().toString();
Tomahawk::InfoSystem::InfoRequestData requestData = reply->property( "requestData" ).value< Tomahawk::InfoSystem::InfoRequestData >();
Tomahawk::InfoSystem::InfoStringHash origData = requestData.input.value< Tomahawk::InfoSystem::InfoStringHash >();
returnedData[ "chart_error" ] = "Invalid ID";
returnedData[ "chart_source" ] = origData[ "chart_source" ];
returnedData[ "chart_id" ] = origData[ "chart_id" ];
emit info( requestData, returnedData );
}
}
qlonglong
ChartsPlugin::getMaxAge( const QByteArray &rawHeader ) const
{
bool ok;
qlonglong expires = QString( rawHeader ).toLongLong( &ok );
if ( ok )
{
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Got rawheader " << QString( rawHeader ) << ":" << expires;
return getMaxAge( expires );
}
return 0;
}
qlonglong
ChartsPlugin::getMaxAge( const qlonglong expires ) const
{
qlonglong currentEpoch = QDateTime::currentMSecsSinceEpoch()/1000;
qlonglong expiresInSeconds = expires-currentEpoch;
if ( expiresInSeconds > 0 )
{
return ( qlonglong )expiresInSeconds*1000;
}
return 0;
}
}

View File

@ -99,9 +99,13 @@ private:
void fetchChartCapabilitiesFromCache( Tomahawk::InfoSystem::InfoRequestData requestData );
void dataError( Tomahawk::InfoSystem::InfoRequestData requestData );
QStringList m_chartResources;
qlonglong getMaxAge( const QByteArray &rawHeader ) const;
qlonglong getMaxAge( const qlonglong expires ) const;
QList< Tomahawk::InfoSystem::InfoStringHash > m_chartResources;
QStringList m_refetchSource;
QString m_chartVersion;
QList< InfoStringHash > m_charts;
ChartType m_chartType;
QVariantMap m_allChartsMap;
uint m_chartsFetchJobs;

View File

@ -175,12 +175,12 @@ void
WhatsHotWidget::fetchData()
{
Tomahawk::InfoSystem::InfoStringHash artistInfo;
Tomahawk::InfoSystem::InfoStringHash criteria;
Tomahawk::InfoSystem::InfoRequestData requestData;
requestData.caller = s_whatsHotIdentifier;
requestData.customData = QVariantMap();
requestData.input = QVariant::fromValue< Tomahawk::InfoSystem::InfoStringHash >( artistInfo );
requestData.input = QVariant::fromValue< Tomahawk::InfoSystem::InfoStringHash >( criteria );
requestData.type = Tomahawk::InfoSystem::InfoChartCapabilities;
requestData.timeoutMillis = 20000;
requestData.allSources = true;
@ -277,9 +277,32 @@ WhatsHotWidget::infoSystemInfo( Tomahawk::InfoSystem::InfoRequestData requestDat
case InfoSystem::InfoChart:
{
if( returnedData.contains( "chart_error") )
{
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Info came back with error!!";
Tomahawk::InfoSystem::InfoStringHash criteria;
criteria.insert( "chart_refetch", returnedData[ "chart_source" ].value< QString >() );
Tomahawk::InfoSystem::InfoRequestData requestData;
requestData.caller = s_whatsHotIdentifier;
requestData.customData = QVariantMap();
requestData.input = QVariant::fromValue< Tomahawk::InfoSystem::InfoStringHash >( criteria );
requestData.type = Tomahawk::InfoSystem::InfoChartCapabilities;
requestData.timeoutMillis = 20000;
requestData.allSources = false;
Tomahawk::InfoSystem::InfoSystem::instance()->getInfo( requestData );
tDebug( LOGVERBOSE ) << "WhatsHot: re-requesting InfoChartCapabilities";
break;
}
if ( !returnedData.contains( "type" ) )
break;
const QString type = returnedData[ "type" ].toString();
if ( !returnedData.contains( type ) )
break;
@ -362,7 +385,6 @@ WhatsHotWidget::leftCrumbIndexChanged( QModelIndex index )
{
tDebug( LOGVERBOSE ) << "WhatsHot: left crumb changed" << index.data();
QStandardItem* item = m_crumbModelLeft->itemFromIndex( m_sortedProxy->mapToSource( index ) );
if ( !item )
return;
@ -382,6 +404,7 @@ WhatsHotWidget::leftCrumbIndexChanged( QModelIndex index )
curr.prepend( index.data().toString().toLower() );
}
const QString chartId = item->data( Breadcrumb::ChartIdRole ).toString();
const qlonglong chartExpires = item->data( Breadcrumb::ChartExpireRole ).toLongLong();
const QString chartSource = curr.takeFirst().toLower();
curr.append( chartSource );
curr.append( chartId );
@ -413,9 +436,9 @@ WhatsHotWidget::leftCrumbIndexChanged( QModelIndex index )
Tomahawk::InfoSystem::InfoStringHash criteria;
criteria.insert( "chart_id", chartId );
criteria.insert( "chart_expires", QString::number( chartExpires ) );
/// Remember to lower the source!
criteria.insert( "chart_source", index.data().toString().toLower() );
Tomahawk::InfoSystem::InfoRequestData requestData;
QVariantMap customData;
customData.insert( "whatshot_side", "left" );
@ -466,6 +489,7 @@ WhatsHotWidget::parseNode( QStandardItem* parentItem, const QString &label, cons
{
QStandardItem *childItem= new QStandardItem( chart[ "label" ] );
childItem->setData( chart[ "id" ], Breadcrumb::ChartIdRole );
childItem->setData( chart[ "expires" ], Breadcrumb::ChartExpireRole );
if ( m_currentVIds.contains( chart.value( "id" ).toLower() ) )
{