1
0
mirror of https://github.com/tomahawk-player/tomahawk.git synced 2025-03-19 15:29:42 +01:00

* Improved file scanning situation a bit.

This commit is contained in:
Christian Muehlhaeuser 2011-11-26 09:10:41 +01:00
parent ba80899ee5
commit 7b96505d27
15 changed files with 88 additions and 182 deletions

View File

@ -342,7 +342,6 @@ EchonestCatalogSynchronizer::tracksRemoved( const QList< unsigned int >& trackId
foreach ( unsigned int id, trackIds )
{
tDebug() << "Deleting item with id:" << id;
Echonest::CatalogUpdateEntry e( Echonest::CatalogTypes::Delete );
e.setItemId( QString::number( id ).toLatin1() );
entries.append( e );

View File

@ -211,7 +211,7 @@ Collection::setStations( const QList< dynplaylist_ptr >& stations )
void
Collection::setTracks( const QList<unsigned int>& ids )
{
qDebug() << Q_FUNC_INFO << ids.count() << name();
tDebug() << Q_FUNC_INFO << ids.count() << name();
m_changed = true;
emit tracksAdded( ids );
@ -221,7 +221,7 @@ Collection::setTracks( const QList<unsigned int>& ids )
void
Collection::delTracks( const QList<unsigned int>& ids )
{
qDebug() << Q_FUNC_INFO << ids.count() << name();
tDebug() << Q_FUNC_INFO << ids.count() << name();
m_changed = true;
emit tracksRemoved( ids );
@ -231,6 +231,7 @@ Collection::delTracks( const QList<unsigned int>& ids )
void
Collection::onSynced()
{
tDebug() << Q_FUNC_INFO << m_changed;
if ( m_changed )
{
m_changed = false;

View File

@ -55,19 +55,12 @@ DatabaseCommand_AddFiles::files() const
void
DatabaseCommand_AddFiles::postCommitHook()
{
if ( source().isNull() || source()->collection().isNull() )
{
qDebug() << "Source has gone offline, not emitting to GUI.";
return;
}
// make the collection object emit its tracksAdded signal, so the
// collection browser will update/fade in etc.
Collection* coll = source()->collection().data();
connect( this, SIGNAL( notify( QList<unsigned int> ) ),
coll, SLOT( setTracks( QList<unsigned int> ) ),
Qt::QueuedConnection );
coll, SLOT( setTracks( QList<unsigned int> ) ), Qt::QueuedConnection );
emit notify( m_ids );
@ -85,23 +78,21 @@ DatabaseCommand_AddFiles::exec( DatabaseImpl* dbi )
TomahawkSqlQuery query_file = dbi->newquery();
TomahawkSqlQuery query_filejoin = dbi->newquery();
TomahawkSqlQuery query_trackattr = dbi->newquery();
TomahawkSqlQuery query_file_del = dbi->newquery();
query_file.prepare( "INSERT INTO file(source, url, size, mtime, md5, mimetype, duration, bitrate) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" );
query_filejoin.prepare( "INSERT INTO file_join(file, artist, album, track, albumpos) VALUES (?, ?, ?, ?, ?)" );
query_trackattr.prepare( "INSERT INTO track_attributes(id, k, v) VALUES (?, ?, ?)" );
query_file_del.prepare( QString( "DELETE FROM file WHERE source %1 AND url = ?" )
.arg( source()->isLocal() ? "IS NULL" : QString( "= %1" ).arg( source()->id() ) ) );
int added = 0;
QVariant srcid = source()->isLocal() ? QVariant( QVariant::Int ) : source()->id();
qDebug() << "Adding" << m_files.length() << "files to db for source" << srcid;
QList<QVariant>::iterator it;
for( it = m_files.begin(); it != m_files.end(); ++it )
for ( it = m_files.begin(); it != m_files.end(); ++it )
{
QVariant& v = *it;
QVariantMap m = v.toMap();
int fileid = 0, artistid = 0, albumid = 0, trackid = 0;
QString url = m.value( "url" ).toString();
int mtime = m.value( "mtime" ).toInt();
@ -116,10 +107,6 @@ DatabaseCommand_AddFiles::exec( DatabaseImpl* dbi )
uint albumpos = m.value( "albumpos" ).toUInt();
int year = m.value( "year" ).toInt();
int fileid = 0, artistid = 0, albumid = 0, trackid = 0;
query_file_del.bindValue( 0, url );
query_file_del.exec();
query_file.bindValue( 0, srcid );
query_file.bindValue( 1, url );
query_file.bindValue( 2, size );
@ -128,28 +115,17 @@ DatabaseCommand_AddFiles::exec( DatabaseImpl* dbi )
query_file.bindValue( 5, mimetype );
query_file.bindValue( 6, duration );
query_file.bindValue( 7, bitrate );
if( !query_file.exec() )
{
qDebug() << "Failed to insert to file:"
<< query_file.lastError().databaseText()
<< query_file.lastError().driverText()
<< query_file.boundValues();
continue;
}
else
{
if( added % 1000 == 0 )
qDebug() << "Inserted" << added;
}
query_file.exec();
if ( added % 1000 == 0 )
qDebug() << "Inserted" << added;
// get internal IDs for art/alb/trk
fileid = query_file.lastInsertId().toInt();
m.insert( "id", fileid );
// this is the qvariant(map) the remote will get
v = m;
if( !source()->isLocal() )
url = QString( "servent://%1\t%2" ).arg( source()->userName() ).arg( url );
artistid = dbi->artistId( artist, true );
if ( artistid < 1 )
continue;
@ -164,44 +140,13 @@ DatabaseCommand_AddFiles::exec( DatabaseImpl* dbi )
query_filejoin.bindValue( 2, albumid > 0 ? albumid : QVariant( QVariant::Int ) );
query_filejoin.bindValue( 3, trackid );
query_filejoin.bindValue( 4, albumpos );
if ( !query_filejoin.exec() )
{
qDebug() << "Error inserting into file_join table";
continue;
}
query_filejoin.exec();
query_trackattr.bindValue( 0, trackid );
query_trackattr.bindValue( 1, "releaseyear" );
query_trackattr.bindValue( 2, year );
query_trackattr.exec();
/* QVariantMap attr;
Tomahawk::query_ptr query = Tomahawk::Query::get( artist, track, album );
attr["releaseyear"] = m.value( "year" );
Tomahawk::artist_ptr artistptr = Tomahawk::Artist::get( artistid, artist );
Tomahawk::album_ptr albumptr = Tomahawk::Album::get( albumid, album, artistptr );
Tomahawk::result_ptr result = Tomahawk::Result::get( url );
result->setModificationTime( mtime );
result->setSize( size );
result->setMimetype( mimetype );
result->setDuration( duration );
result->setBitrate( bitrate );
result->setArtist( artistptr );
result->setAlbum( albumptr );
result->setTrack( track );
result->setAlbumPos( albumpos );
result->setAttributes( attr );
result->setCollection( source()->collection() );
result->setScore( 1.0 );
result->setId( trackid );
QList<Tomahawk::result_ptr> results;
results << result;
query->addResults( results );
m_queries << query;*/
m_ids << fileid;
added++;
}
@ -210,6 +155,6 @@ DatabaseCommand_AddFiles::exec( DatabaseImpl* dbi )
if ( added )
source()->updateIndexWhenSynced();
qDebug() << "Committing" << added << "tracks...";
tDebug() << "Committing" << added << "tracks...";
emit done( m_files, source()->collection() );
}

View File

@ -37,7 +37,7 @@ using namespace Tomahawk;
void
DatabaseCommand_DeleteFiles::postCommitHook()
{
if ( !m_files.count() )
if ( !m_idList.count() )
return;
// make the collection object emit its tracksAdded signal, so the
@ -47,11 +47,8 @@ DatabaseCommand_DeleteFiles::postCommitHook()
connect( this, SIGNAL( notify( QList<unsigned int> ) ),
coll, SLOT( delTracks( QList<unsigned int> ) ), Qt::QueuedConnection );
tDebug() << "Notifying of deleted tracks:" << m_ids.size() << "from source" << source()->id();
QList<unsigned int> ids;
foreach ( QVariant id, m_ids )
ids << id.toUInt();
emit notify( ids );
tDebug() << "Notifying of deleted tracks:" << m_idList.size() << "from source" << source()->id();
emit notify( m_idList );
if ( source()->isLocal() )
Servent::instance()->triggerDBSync();
@ -65,111 +62,78 @@ DatabaseCommand_DeleteFiles::exec( DatabaseImpl* dbi )
QVariant srcid = source()->isLocal() ? QVariant( QVariant::Int ) : source()->id();
TomahawkSqlQuery delquery = dbi->newquery();
QString lastPath;
if ( source()->isLocal() )
if ( m_deleteAll )
{
TomahawkSqlQuery dirquery = dbi->newquery();
dirquery.prepare( QString( "SELECT id FROM file WHERE source %1" )
.arg( source()->isLocal() ? "IS NULL" : QString( "= %1" ).arg( source()->id() ) ) );
dirquery.exec();
while ( dirquery.next() )
m_idList << dirquery.value( 0 ).toUInt();
}
else if ( source()->isLocal() )
{
if ( m_dir.path() != QString( "." ) )
{
tDebug() << "Deleting" << m_dir.path() << "from db for localsource" << srcid;
TomahawkSqlQuery dirquery = dbi->newquery();
QString path( "file://" + m_dir.canonicalPath() + "/%" );
dirquery.prepare( QString( "SELECT id, url FROM file WHERE source IS NULL AND url LIKE '%1'" ).arg( TomahawkUtils::sqlEscape( path ) ) );
dirquery.prepare( QString( "SELECT id FROM file WHERE source IS NULL AND url LIKE '%1'" ).arg( TomahawkUtils::sqlEscape( path ) ) );
dirquery.exec();
while ( dirquery.next() )
{
QFileInfo fi( dirquery.value( 1 ).toString().mid( 7 ) ); // remove file://
if ( fi.canonicalPath() != m_dir.canonicalPath() )
{
if ( lastPath != fi.canonicalPath() )
tDebug() << "Skipping subdir:" << fi.canonicalPath();
lastPath = fi.canonicalPath();
continue;
}
m_files << dirquery.value( 1 ).toString();
m_ids << dirquery.value( 0 ).toUInt();
m_ids << dirquery.value( 0 );
m_idList << dirquery.value( 0 ).toUInt();
}
}
else if ( !m_ids.isEmpty() )
{
tDebug() << Q_FUNC_INFO << "deleting given ids";
TomahawkSqlQuery dirquery = dbi->newquery();
QString idstring;
foreach( const QVariant& id, m_ids )
idstring.append( id.toString() + ", " );
idstring.chop( 2 ); //remove the trailing ", "
dirquery.prepare( QString( "SELECT id, url FROM file WHERE source IS NULL AND id IN ( %1 )" ).arg( idstring ) );
dirquery.exec();
while ( dirquery.next() )
{
m_files << dirquery.value( 1 ).toString();
}
foreach ( const QVariant& id, m_ids )
m_idList << id.toUInt();
}
else if ( m_deleteAll )
{
TomahawkSqlQuery dirquery = dbi->newquery();
dirquery.prepare( QString( "SELECT id, url FROM file WHERE source IS NULL" ) );
dirquery.exec();
while ( dirquery.next() )
{
m_ids << dirquery.value( 0 ).toString();
m_files << dirquery.value( 1 ).toString();
}
}
}
else
{
if ( m_deleteAll )
{
TomahawkSqlQuery dirquery = dbi->newquery();
dirquery.prepare( QString( "SELECT url FROM file WHERE source = %1" ).arg( source()->id() ) );
dirquery.exec();
while ( dirquery.next() )
m_ids << dirquery.value( 0 ).toString();
}
foreach( const QVariant& id, m_ids )
m_files << QString( "servent://%1\t%2" ).arg( source()->userName() ).arg( id.toString() );
}
if ( m_deleteAll )
{
if ( !m_ids.isEmpty() )
{
delquery.prepare( QString( "DELETE FROM file WHERE source %1" )
.arg( source()->isLocal() ? "IS NULL" : QString( "= %1" ).arg( source()->id() ) ) );
delquery.exec();
}
delquery.prepare( QString( "DELETE FROM file WHERE source %1" )
.arg( source()->isLocal() ? "IS NULL" : QString( "= %1" ).arg( source()->id() ) ) );
delquery.exec();
}
else if ( !m_ids.isEmpty() )
{
delquery.prepare( QString( "DELETE FROM file WHERE source %1 AND %2 IN ( ? )" )
.arg( source()->isLocal() ? "IS NULL" : QString( "= %1" ).arg( source()->id() ) )
.arg( source()->isLocal() ? "id" : "url" ) );
QString idstring;
foreach( const QVariant& id, m_ids )
foreach ( const QVariant& id, m_ids )
idstring.append( id.toString() + ", " );
idstring.chop( 2 ); //remove the trailing ", "
delquery.prepare( QString( "DELETE FROM file WHERE source %1 AND %2 IN ( %3 )" )
if ( !source()->isLocal() )
{
delquery.prepare( QString( "SELECT id FROM file WHERE source = %1 AND url IN ( %2 )" )
.arg( source()->id() )
.arg( idstring ) );
idstring = QString();
while ( delquery.next() )
{
idstring.append( delquery.value( 0 ).toString() + ", " );
m_idList << delquery.value( 0 ).toUInt();
}
idstring.chop( 2 ); //remove the trailing ", "
}
delquery.prepare( QString( "DELETE FROM file WHERE source %1 AND id IN ( %2 )" )
.arg( source()->isLocal() ? "IS NULL" : QString( "= %1" ).arg( source()->id() ) )
.arg( source()->isLocal() ? "id" : "url" )
.arg( idstring ) );
delquery.exec();
}
emit done( m_files, source()->collection() );
if ( m_idList.count() )
source()->updateIndexWhenSynced();
emit done( m_idList, source()->collection() );
}

View File

@ -74,13 +74,13 @@ public:
void setDeleteAll( const bool deleteAll ) { m_deleteAll = deleteAll; }
signals:
void done( const QStringList&, const Tomahawk::collection_ptr& );
void done( const QList<unsigned int>&, const Tomahawk::collection_ptr& );
void notify( const QList<unsigned int>& ids );
private:
QStringList m_files;
QDir m_dir;
QVariantList m_ids;
QList<unsigned int> m_idList;
bool m_deleteAll;
};

View File

@ -40,8 +40,8 @@ DatabaseCommand_FileMtimes::execSelect( DatabaseImpl* dbi )
TomahawkSqlQuery query = dbi->newquery();
if( m_prefix.isEmpty() && m_prefixes.isEmpty() )
{
QString limit( m_checkonly ? QString( " LIMIT 1" ) : QString() );
query.exec( QString( "SELECT url, id, mtime FROM file WHERE source IS NULL%1" ).arg( limit ) );
QString limit( m_checkonly ? QString( "LIMIT 1" ) : QString() );
query.exec( QString( "SELECT url, id, mtime FROM file WHERE source IS NULL %1" ).arg( limit ) );
while( query.next() )
{
QMap< unsigned int, unsigned int > map;

View File

@ -509,7 +509,6 @@ DropJob::handleXspfs( const QString& fileUrls )
{
qDebug() << Q_FUNC_INFO << "Trying to append XSPF";
connect( l, SIGNAL( tracks( QList<Tomahawk::query_ptr> ) ), this, SLOT( onTracksAdded( QList< Tomahawk::query_ptr > ) ) );
}
m_queryCount++;

View File

@ -309,12 +309,12 @@ AlbumModel::addAlbums( const QList<Tomahawk::album_ptr>& albums )
{
emit loadingFinished();
if ( !albums.count() )
return;
if ( m_overwriteOnAdd )
clear();
if ( !albums.count() )
return;
int c = rowCount( QModelIndex() );
QPair< int, int > crows;
crows.first = c;

View File

@ -151,6 +151,7 @@ TreeProxyModel::onFilterArtists( const QList<Tomahawk::artist_ptr>& artists )
{
bool finished = true;
m_artistsFilter = artists;
m_artistsFilterCmd = 0;
foreach ( const Tomahawk::artist_ptr& artist, artists )
{

View File

@ -435,8 +435,7 @@ Source::updateTracks()
{
// Re-calculate local db stats
DatabaseCommand_CollectionStats* cmd = new DatabaseCommand_CollectionStats( SourceList::instance()->get( id() ) );
connect( cmd, SIGNAL( done( QVariantMap ) ),
this, SLOT( setStats( QVariantMap ) ), Qt::QueuedConnection );
connect( cmd, SIGNAL( done( QVariantMap ) ), SLOT( setStats( QVariantMap ) ), Qt::QueuedConnection );
Database::instance()->enqueue( QSharedPointer<DatabaseCommand>( cmd ) );
}
}

View File

@ -34,6 +34,7 @@ class ControlConnection;
class DatabaseCommand_LogPlayback;
class DatabaseCommand_SocialAction;
class DatabaseCommand_UpdateSearchIndex;
class DatabaseCommand_DeleteFiles;
namespace Tomahawk
{
@ -47,6 +48,7 @@ friend class ::ControlConnection;
friend class ::DatabaseCommand_LogPlayback;
friend class ::DatabaseCommand_SocialAction;
friend class ::DatabaseCommand_AddFiles;
friend class ::DatabaseCommand_DeleteFiles;
public:
enum AvatarStyle { Original, FancyStyle };

View File

@ -91,7 +91,10 @@ DirLister::scanDir( QDir dir, int depth )
m_opcount--;
if ( m_opcount == 0 )
{
tDebug() << Q_FUNC_INFO << "emitting finished";
emit finished();
}
}
@ -133,7 +136,7 @@ void
MusicScanner::startScan()
{
tDebug( LOGVERBOSE ) << "Loading mtimes...";
m_scanned = m_skipped = 0;
m_scanned = m_skipped = m_cmdQueue = 0;
m_skippedFiles.clear();
// trigger the scan once we've loaded old filemtimes
@ -192,10 +195,11 @@ MusicScanner::listerFinished()
foreach( const QString& key, m_filemtimes.keys() )
m_filesToDelete << m_filemtimes[ key ].keys().first();
commitBatch( m_scannedfiles, m_filesToDelete );
tDebug() << "Lister finished: to delete:" << m_filesToDelete;
if ( m_filesToDelete.length() || m_scannedfiles.length() )
{
commitBatch( m_scannedfiles, m_filesToDelete );
m_scannedfiles.clear();
m_filesToDelete.clear();
@ -222,6 +226,7 @@ MusicScanner::cleanup()
m_dirListerThreadController = 0;
}
tDebug() << Q_FUNC_INFO << "emitting finished!";
emit finished();
}
@ -232,15 +237,13 @@ MusicScanner::commitBatch( const QVariantList& tracks, const QVariantList& delet
if ( deletethese.length() )
{
tDebug( LOGINFO ) << Q_FUNC_INFO << "deleting" << deletethese.length() << "tracks";
source_ptr localsrc = SourceList::instance()->getLocal();
executeCommand( QSharedPointer<DatabaseCommand>( new DatabaseCommand_DeleteFiles( deletethese, SourceList::instance()->getLocal() ) ) );
}
if ( tracks.length() )
{
tDebug( LOGINFO ) << Q_FUNC_INFO << "adding" << tracks.length() << "tracks";
source_ptr localsrc = SourceList::instance()->getLocal();
executeCommand( QSharedPointer<DatabaseCommand>( new DatabaseCommand_AddFiles( tracks, localsrc ) ) );
executeCommand( QSharedPointer<DatabaseCommand>( new DatabaseCommand_AddFiles( tracks, SourceList::instance()->getLocal() ) ) );
}
}
@ -248,7 +251,8 @@ MusicScanner::commitBatch( const QVariantList& tracks, const QVariantList& delet
void
MusicScanner::executeCommand( QSharedPointer< DatabaseCommand > cmd )
{
m_cmdQueue << cmd.data();
tDebug() << Q_FUNC_INFO << m_cmdQueue;
m_cmdQueue++;
connect( cmd.data(), SIGNAL( finished() ), SLOT( commandFinished() ) );
Database::instance()->enqueue( cmd );
}
@ -257,10 +261,9 @@ MusicScanner::executeCommand( QSharedPointer< DatabaseCommand > cmd )
void
MusicScanner::commandFinished()
{
DatabaseCommand* cmd = qobject_cast< DatabaseCommand* >( sender() );
m_cmdQueue.removeAll( cmd );
tDebug() << Q_FUNC_INFO << m_cmdQueue;
if ( m_cmdQueue.isEmpty() )
if ( --m_cmdQueue == 0 )
cleanup();
}
@ -268,7 +271,6 @@ MusicScanner::commandFinished()
void
MusicScanner::scanFile( const QFileInfo& fi )
{
tDebug() << Q_FUNC_INFO << "scanning file:" << fi.canonicalFilePath();
if ( m_filemtimes.contains( "file://" + fi.canonicalFilePath() ) )
{
if ( fi.lastModified().toUTC().toTime_t() == m_filemtimes.value( "file://" + fi.canonicalFilePath() ).values().first() )
@ -281,6 +283,7 @@ MusicScanner::scanFile( const QFileInfo& fi )
m_filemtimes.remove( "file://" + fi.canonicalFilePath() );
}
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Scanning file:" << fi.canonicalFilePath();
QVariant m = readFile( fi );
if ( m.toMap().isEmpty() )
return;
@ -302,7 +305,6 @@ MusicScanner::readFile( const QFileInfo& fi )
if ( !m_ext2mime.contains( suffix ) )
{
m_skipped++;
return QVariantMap(); // invalid extension
}

View File

@ -113,7 +113,7 @@ private:
QList<QString> m_skippedFiles;
QMap<QString, QMap< unsigned int, unsigned int > > m_filemtimes;
QList<DatabaseCommand*> m_cmdQueue;
unsigned int m_cmdQueue;
QVariantList m_scannedfiles;
QVariantList m_filesToDelete;

View File

@ -140,9 +140,7 @@ ScanManager::runScan( bool manualFull )
if ( manualFull )
{
DatabaseCommand_DeleteFiles *cmd = new DatabaseCommand_DeleteFiles( SourceList::instance()->getLocal() );
connect( cmd, SIGNAL( done( const QStringList&, const Tomahawk::collection_ptr& ) ),
SLOT( filesDeleted( const QStringList&, const Tomahawk::collection_ptr& ) ) );
connect( cmd, SIGNAL( finished() ), SLOT( filesDeleted() ) );
Database::instance()->enqueue( QSharedPointer< DatabaseCommand >( cmd ) );
return;
}
@ -167,9 +165,7 @@ ScanManager::fileMtimesCheck( const QMap< QString, QMap< unsigned int, unsigned
if ( !mtimes.isEmpty() && TomahawkSettings::instance()->scannerPaths().isEmpty() )
{
DatabaseCommand_DeleteFiles *cmd = new DatabaseCommand_DeleteFiles( SourceList::instance()->getLocal() );
connect( cmd, SIGNAL( done( const QStringList&, const Tomahawk::collection_ptr& ) ),
SLOT( filesDeleted( const QStringList&, const Tomahawk::collection_ptr& ) ) );
connect( cmd, SIGNAL( finished() ), SLOT( filesDeleted() ) );
Database::instance()->enqueue( QSharedPointer< DatabaseCommand >( cmd ) );
return;
}
@ -179,12 +175,12 @@ ScanManager::fileMtimesCheck( const QMap< QString, QMap< unsigned int, unsigned
void
ScanManager::filesDeleted( const QStringList& files, const Tomahawk::collection_ptr& collection )
ScanManager::filesDeleted()
{
Q_UNUSED( files );
Q_UNUSED( collection );
if ( !TomahawkSettings::instance()->scannerPaths().isEmpty() )
runDirScan();
else
scannerFinished();
}
@ -193,9 +189,6 @@ ScanManager::runDirScan()
{
qDebug() << Q_FUNC_INFO;
if ( !Database::instance() || ( Database::instance() && !Database::instance()->isReady() ) )
return;
QStringList paths = TomahawkSettings::instance()->scannerPaths();
if ( !m_musicScannerThreadController && m_scanner.isNull() ) //still running if these are not zero
@ -219,6 +212,7 @@ ScanManager::runDirScan()
void
ScanManager::scannerFinished()
{
tDebug() << "deleting scanner";
if ( !m_scanner.isNull() )
{
m_musicScannerThreadController->quit();

View File

@ -59,7 +59,7 @@ private slots:
void onSettingsChanged();
void fileMtimesCheck( const QMap< QString, QMap< unsigned int, unsigned int > >& mtimes );
void filesDeleted( const QStringList& files, const Tomahawk::collection_ptr& collection );
void filesDeleted();
private:
static ScanManager* s_instance;