From 4aefc4ecbff00c4da5525d7288062cfc8cca973e Mon Sep 17 00:00:00 2001 From: Christian Muehlhaeuser Date: Thu, 3 Feb 2011 11:02:35 +0100 Subject: [PATCH] * Tomahawk now relies on Lucene for indexing / searching Artists, Albums and Tracks. * Lucene index is stored on disk for faster startups. * Queries are now auto-resolved when retrieved via Query::get( QVariant, true ). The second parameter is optional and true by default. * Updated database schema and removed the old ngram tables. * Hopefully didn't break too much :-) --- CMakeLists.txt | 1 + src/libtomahawk/CMakeLists.txt | 3 + src/libtomahawk/database/database.h | 7 +- .../database/databasecommand_addfiles.cpp | 26 +- .../database/databasecommand_alltracks.cpp | 2 +- .../databasecommand_loadplaylistentries.cpp | 2 +- .../database/databasecommand_logplayback.cpp | 5 +- .../databasecommand_playbackhistory.cpp | 7 +- .../database/databasecommand_resolve.cpp | 12 +- .../database/databasecommand_resolve.h | 3 +- .../databasecommand_updatesearchindex.cpp | 128 ++-------- .../databasecommand_updatesearchindex.h | 10 +- src/libtomahawk/database/databaseimpl.cpp | 85 ++----- src/libtomahawk/database/databaseimpl.h | 7 +- src/libtomahawk/database/databaseresolver.cpp | 13 +- src/libtomahawk/database/databaseresolver.h | 3 +- src/libtomahawk/database/databaseworker.cpp | 9 +- src/libtomahawk/database/fuzzyindex.cpp | 225 +++++++++--------- src/libtomahawk/database/fuzzyindex.h | 46 +++- src/libtomahawk/database/gen_schema.h.sh | 4 + src/libtomahawk/database/schema.sql | 48 +--- src/libtomahawk/database/schema.sql.h | 22 +- src/libtomahawk/pipeline.cpp | 34 ++- src/libtomahawk/pipeline.h | 10 +- src/libtomahawk/playlist.cpp | 5 +- src/libtomahawk/playlist/collectionmodel.cpp | 2 +- src/libtomahawk/query.cpp | 34 ++- src/libtomahawk/query.h | 10 +- src/libtomahawk/result.cpp | 2 +- src/libtomahawk/utils/xspfloader.cpp | 2 +- src/libtomahawk/widgets/newplaylistwidget.cpp | 3 - src/tomahawkapp.cpp | 3 +- src/tomahawkwindow.cpp | 2 +- src/web/api_v1.h | 3 +- src/xmppbot/xmppbot.cpp | 4 +- 35 files changed, 338 insertions(+), 444 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 44ba445be..3292a8c77 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,6 +16,7 @@ ENDIF() FIND_PACKAGE( Taglib 1.6.0 REQUIRED ) FIND_PACKAGE( LibLastFm 0.3.3 REQUIRED ) FIND_PACKAGE( LibEchonest REQUIRED ) +FIND_PACKAGE( CLucene REQUIRED ) IF( UNIX AND NOT APPLE ) ADD_SUBDIRECTORY( alsa-playback ) diff --git a/src/libtomahawk/CMakeLists.txt b/src/libtomahawk/CMakeLists.txt index d33b1dde0..1ce0be19e 100644 --- a/src/libtomahawk/CMakeLists.txt +++ b/src/libtomahawk/CMakeLists.txt @@ -228,6 +228,8 @@ set( libUI ${libUI} include_directories( . ${CMAKE_CURRENT_BINARY_DIR} .. ${QT_INCLUDE_DIR} + ${CLUCENE_INCLUDE_DIR} + ${CLUCENE_LIBRARY_DIR} ../../libportfwd/include ../../include ../network @@ -296,6 +298,7 @@ target_link_libraries( tomahawklib ogg FLAC++ jdns + clucene ) install( TARGETS tomahawklib DESTINATION lib ) diff --git a/src/libtomahawk/database/database.h b/src/libtomahawk/database/database.h index 156ba6743..f088b9648 100644 --- a/src/libtomahawk/database/database.h +++ b/src/libtomahawk/database/database.h @@ -16,10 +16,9 @@ HOWEVER, we're using the command pattern to serialize access to the database and provide an async api. You create a DatabaseCommand object, and add it to - the queue of work. There is a single thread responsible for exec'ing all - the commands, so sqlite only does one thing at a time. - - Update: 1 thread for mutates, one for readonly queries. + the queue of work. There is a threadpool responsible for exec'ing all + the non-mutating (readonly) commands and one separate thread for mutating ones, + so sqlite doesn't write to the Database from multiple threads. */ class DLLEXPORT Database : public QObject { diff --git a/src/libtomahawk/database/databasecommand_addfiles.cpp b/src/libtomahawk/database/databasecommand_addfiles.cpp index 0426d8638..1f9485f87 100644 --- a/src/libtomahawk/database/databasecommand_addfiles.cpp +++ b/src/libtomahawk/database/databasecommand_addfiles.cpp @@ -16,7 +16,7 @@ QVariantList DatabaseCommand_AddFiles::files() const { QVariantList list; - foreach( const QVariant& v, m_files ) + foreach ( const QVariant& v, m_files ) { // replace url with the id, we don't leak file paths over the network. QVariantMap m = v.toMap(); @@ -73,11 +73,7 @@ DatabaseCommand_AddFiles::exec( DatabaseImpl* dbi ) query_trackattr.prepare( "INSERT INTO track_attributes(id, k, v) " "VALUES (?,?,?)" ); query_file_del.prepare( QString( "DELETE FROM file WHERE source %1 AND url = ?" ) - .arg( source()->isLocal() ? "IS NULL" : QString( "= %1" ).arg( source()->id() ) - ) ); - - int maxart, maxalb, maxtrk; // store max id, so we can index new ones after - maxart = maxalb = maxtrk = 0; + .arg( source()->isLocal() ? "IS NULL" : QString( "= %1" ).arg( source()->id() ) ) ); int added = 0; QVariant srcid = source()->isLocal() ? @@ -126,29 +122,28 @@ DatabaseCommand_AddFiles::exec( DatabaseImpl* dbi ) } else { - if( added % 100 == 0 ) qDebug() << "Inserted" << added; + if( added % 100 == 0 ) + qDebug() << "Inserted" << added; } // get internal IDs for art/alb/trk fileid = query_file.lastInsertId().toInt(); // insert the new fileid, set the url for our use: m.insert( "id", fileid ); - if( !source()->isLocal() ) m["url"] = QString( "servent://%1\t%2" ) - .arg( source()->userName() ) - .arg( fileid ); + if( !source()->isLocal() ) + m["url"] = QString( "servent://%1\t%2" ) + .arg( source()->userName() ) + .arg( fileid ); v = m; bool isnew; int artid = dbi->artistId( artist, isnew ); if( artid < 1 ) continue; - if( isnew && maxart == 0 ) maxart = artid; int trkid = dbi->trackId( artid, track, isnew ); if( trkid < 1 ) continue; - if( isnew && maxtrk == 0 ) maxtrk = trkid; int albid = dbi->albumId( artid, album, isnew ); - if( albid > 0 && isnew && maxalb == 0 ) maxalb = albid; // Now add the association query_filejoin.bindValue( 0, fileid ); @@ -172,11 +167,8 @@ DatabaseCommand_AddFiles::exec( DatabaseImpl* dbi ) qDebug() << "Inserted" << added; // TODO building the index could be a separate job, outside this transaction - if(maxart) dbi->updateSearchIndex( "artist", maxart ); - if(maxalb) dbi->updateSearchIndex( "album", maxalb ); - if(maxtrk) dbi->updateSearchIndex( "track", maxtrk ); + dbi->updateSearchIndex(); qDebug() << "Committing" << added << "tracks..."; - qDebug() << "Done."; emit done( m_files, source()->collection() ); } diff --git a/src/libtomahawk/database/databasecommand_alltracks.cpp b/src/libtomahawk/database/databasecommand_alltracks.cpp index 8d4fd9522..13b721c0d 100644 --- a/src/libtomahawk/database/databasecommand_alltracks.cpp +++ b/src/libtomahawk/database/databasecommand_alltracks.cpp @@ -89,7 +89,7 @@ DatabaseCommand_AllTracks::exec( DatabaseImpl* dbi ) attr[ attrQuery.value( 0 ).toString() ] = attrQuery.value( 1 ).toString(); } - Tomahawk::query_ptr query = Tomahawk::query_ptr( new Tomahawk::Query( t ) ); + Tomahawk::query_ptr query = Tomahawk::Query::get( t, false ); t["score"] = 1.0; Tomahawk::result_ptr result = Tomahawk::result_ptr( new Tomahawk::Result( t, m_collection ) ); diff --git a/src/libtomahawk/database/databasecommand_loadplaylistentries.cpp b/src/libtomahawk/database/databasecommand_loadplaylistentries.cpp index 60c21b3af..a214ad0a2 100644 --- a/src/libtomahawk/database/databasecommand_loadplaylistentries.cpp +++ b/src/libtomahawk/database/databasecommand_loadplaylistentries.cpp @@ -60,7 +60,7 @@ DatabaseCommand_LoadPlaylistEntries::exec( DatabaseImpl* dbi ) m.insert( "resulthint", query.value( 8 ).toString() ); m.insert( "qid", uuid() ); - Tomahawk::query_ptr q( new Tomahawk::Query( m ) ); + Tomahawk::query_ptr q = Tomahawk::Query::get( m, false ); e->setQuery( q ); entrymap.insert( e->guid(), e ); diff --git a/src/libtomahawk/database/databasecommand_logplayback.cpp b/src/libtomahawk/database/databasecommand_logplayback.cpp index 1aa617859..5ce6976a8 100644 --- a/src/libtomahawk/database/databasecommand_logplayback.cpp +++ b/src/libtomahawk/database/databasecommand_logplayback.cpp @@ -5,7 +5,6 @@ #include "collection.h" #include "database/database.h" #include "databaseimpl.h" -#include "pipeline.h" #include "network/servent.h" using namespace Tomahawk; @@ -26,9 +25,7 @@ DatabaseCommand_LogPlayback::postCommitHook() m.insert( "track", m_track ); m.insert( "artist", m_artist ); m.insert( "qid", uuid() ); - Tomahawk::query_ptr q( new Tomahawk::Query( m ) ); - - Tomahawk::Pipeline::instance()->add( q ); + Tomahawk::query_ptr q = Tomahawk::Query::get( m ); if ( m_action == Finished ) { diff --git a/src/libtomahawk/database/databasecommand_playbackhistory.cpp b/src/libtomahawk/database/databasecommand_playbackhistory.cpp index d6727a4e8..e7f7b8979 100644 --- a/src/libtomahawk/database/databasecommand_playbackhistory.cpp +++ b/src/libtomahawk/database/databasecommand_playbackhistory.cpp @@ -3,7 +3,7 @@ #include #include "databaseimpl.h" -#include "pipeline.h" + void DatabaseCommand_PlaybackHistory::exec( DatabaseImpl* dbi ) @@ -49,7 +49,7 @@ DatabaseCommand_PlaybackHistory::exec( DatabaseImpl* dbi ) m.insert( "artist", query_track.value( 1 ).toString() ); m.insert( "qid", uuid() ); - Tomahawk::query_ptr q( new Tomahawk::Query( m ) ); + Tomahawk::query_ptr q = Tomahawk::Query::get( m ); ql << q; } } @@ -57,8 +57,5 @@ DatabaseCommand_PlaybackHistory::exec( DatabaseImpl* dbi ) qDebug() << Q_FUNC_INFO << ql.length(); if ( ql.count() ) - { - Tomahawk::Pipeline::instance()->add( ql ); emit tracks( ql ); - } } diff --git a/src/libtomahawk/database/databasecommand_resolve.cpp b/src/libtomahawk/database/databasecommand_resolve.cpp index 20b78dab5..7cf8ac5cd 100644 --- a/src/libtomahawk/database/databasecommand_resolve.cpp +++ b/src/libtomahawk/database/databasecommand_resolve.cpp @@ -8,10 +8,9 @@ using namespace Tomahawk; -DatabaseCommand_Resolve::DatabaseCommand_Resolve( const QVariant& v, bool searchlocal ) +DatabaseCommand_Resolve::DatabaseCommand_Resolve( const QVariant& v ) : DatabaseCommand() , m_v( v ) - , m_searchlocal( searchlocal ) { } @@ -72,6 +71,7 @@ DatabaseCommand_Resolve::exec( DatabaseImpl* lib ) if( artists.length() == 0 || tracks.length() == 0 ) { //qDebug() << "No candidates found in first pass, aborting resolve" << artistname << trackname; + emit results( qid, res ); return; } @@ -98,11 +98,9 @@ DatabaseCommand_Resolve::exec( DatabaseImpl* lib ) "WHERE " "artist.id = file_join.artist AND " "track.id = file_join.track AND " - "file.source %1 AND " "file.id = file_join.file AND " - "file_join.artist IN (%2) AND " - "file_join.track IN (%3)" - ).arg( m_searchlocal ? "IS NULL" : " > 0 " ) + "file_join.artist IN (%1) AND " + "file_join.track IN (%2)" ) .arg( artsl.join( "," ) ) .arg( trksl.join( "," ) ); @@ -130,7 +128,7 @@ DatabaseCommand_Resolve::exec( DatabaseImpl* lib ) source_ptr s; const QString url_str = files_query.value( 0 ).toString(); - if( m_searchlocal ) + if( files_query.value( 13 ).toUInt() == 0 ) { s = SourceList::instance()->getLocal(); m["url"] = url_str; diff --git a/src/libtomahawk/database/databasecommand_resolve.h b/src/libtomahawk/database/databasecommand_resolve.h index 78015f920..e62ce076a 100644 --- a/src/libtomahawk/database/databasecommand_resolve.h +++ b/src/libtomahawk/database/databasecommand_resolve.h @@ -13,7 +13,7 @@ class DLLEXPORT DatabaseCommand_Resolve : public DatabaseCommand { Q_OBJECT public: - explicit DatabaseCommand_Resolve( const QVariant& v, bool searchlocal ); + explicit DatabaseCommand_Resolve( const QVariant& v ); virtual QString commandname() const { return "dbresolve"; } virtual bool doesMutates() const { return false; } @@ -27,7 +27,6 @@ public slots: private: QVariant m_v; - bool m_searchlocal; float how_similar( const QVariantMap& q, const QVariantMap& r ); static int levenshtein( const QString& source, const QString& target ); diff --git a/src/libtomahawk/database/databasecommand_updatesearchindex.cpp b/src/libtomahawk/database/databasecommand_updatesearchindex.cpp index f4c517ffb..67d9e9b60 100644 --- a/src/libtomahawk/database/databasecommand_updatesearchindex.cpp +++ b/src/libtomahawk/database/databasecommand_updatesearchindex.cpp @@ -1,117 +1,39 @@ #include "databasecommand_updatesearchindex.h" -DatabaseCommand_UpdateSearchIndex::DatabaseCommand_UpdateSearchIndex( const QString& t, int p ) +DatabaseCommand_UpdateSearchIndex::DatabaseCommand_UpdateSearchIndex() : DatabaseCommand() - , table( t ) - , pkey( p ) { - if( table != "artist" && table != "track" && table != "album" ) - { - Q_ASSERT(false); - return; - } } -void DatabaseCommand_UpdateSearchIndex::exec(DatabaseImpl *db) +void +DatabaseCommand_UpdateSearchIndex::indexTable( DatabaseImpl* db, const QString& table ) { qDebug() << Q_FUNC_INFO; - - if( table != "artist" && table != "track" && table != "album" ) - { - Q_ASSERT(false); - return; - } - - // if pkey is 0, consult DB to see what needs indexing - if( pkey == 0 ) - { - TomahawkSqlQuery q = db->newquery(); - q.exec( QString("SELECT coalesce(max(id),0) from %1_search_index").arg(table) ); - q.next(); - pkey = 1 + q.value(0).toInt(); - qDebug() << "updateSearchIndex" << table << "consulted DB, starting at" << pkey; - } - + TomahawkSqlQuery query = db->newquery(); - qDebug() << "Building index for" << table << ">= id" << pkey; - QString searchtable( table + "_search_index" ); - query.exec(QString( "SELECT id, sortname FROM %1 WHERE id >= %2" ).arg( table ).arg(pkey ) ); - - TomahawkSqlQuery upq = db->newquery(); - TomahawkSqlQuery inq = db->newquery(); - inq.prepare( "INSERT INTO "+ searchtable +" (ngram, id, num) VALUES (?,?,?)" ); - upq.prepare( "UPDATE "+ searchtable +" SET num=num+? WHERE ngram=? AND id=?" ); - - int num_names = 0; - int num_ngrams = 0; - int id; - QString name; - QMap ngrammap; - - // this is the new ngram map we build up, to be merged into the - // main one in FuzzyIndex: - QHash< QString, QMap > idx; - - while( query.next() ) + qDebug() << "Building index for" << table; + query.exec( QString( "SELECT id, name FROM %1" ).arg( table ) ); + + QMap< unsigned int, QString > fields; + while ( query.next() ) { - id = query.value( 0 ).toInt(); - name = query.value( 1 ).toString(); - num_names++; - inq.bindValue( 1, id ); // set id - upq.bindValue( 2, id ); // set id - ngrammap = DatabaseImpl::ngrams( name ); - QMapIterator i( ngrammap ); - - while ( i.hasNext() ) - { - i.next(); - num_ngrams++; - upq.bindValue( 0, i.value() ); //num - upq.bindValue( 1, i.key() ); // ngram - upq.exec(); - - if( upq.numRowsAffected() == 0 ) - { - inq.bindValue( 0, i.key() ); //ngram - inq.bindValue( 2, i.value() ); //num - inq.exec(); - if( inq.numRowsAffected() == 0 ) - { - qDebug() << "Error updating search index:" << id << name; - continue; - } - } - - // update ngram cache: - QMapIterator iter( ngrammap ); - while ( iter.hasNext() ) - { - iter.next(); - if( idx.contains( iter.key() ) ) - { - idx[ iter.key() ][ id ] += iter.value(); - } - else - { - QMap tmp; - tmp.insert( id, iter.value() ); - idx.insert( iter.key(), tmp ); - } - } - - - } + fields.insert( query.value( 0 ).toUInt(), query.value( 1 ).toString() ); } - - // merge in our ngrams into the main index - QMetaObject::invokeMethod( &(db->m_fuzzyIndex), - "mergeIndex", - Qt::QueuedConnection, - Q_ARG( QString, table ), - QGenericArgument( "QHash< QString, QMap >", &idx ) - ); - - qDebug() << "Finished indexing" << num_names <<" names," << num_ngrams << "ngrams."; + + db->m_fuzzyIndex->appendFields( table, fields ); +} + + +void +DatabaseCommand_UpdateSearchIndex::exec( DatabaseImpl* db ) +{ + db->m_fuzzyIndex->beginIndexing(); + + indexTable( db, "artist" ); + indexTable( db, "album" ); + indexTable( db, "track" ); + + db->m_fuzzyIndex->endIndexing(); } diff --git a/src/libtomahawk/database/databasecommand_updatesearchindex.h b/src/libtomahawk/database/databasecommand_updatesearchindex.h index c9278e0cb..d8d74ed02 100644 --- a/src/libtomahawk/database/databasecommand_updatesearchindex.h +++ b/src/libtomahawk/database/databasecommand_updatesearchindex.h @@ -10,21 +10,19 @@ class DLLEXPORT DatabaseCommand_UpdateSearchIndex : public DatabaseCommand { Q_OBJECT public: - explicit DatabaseCommand_UpdateSearchIndex(const QString& table, int pkey); + explicit DatabaseCommand_UpdateSearchIndex(); virtual QString commandname() const { return "updatesearchindex"; } virtual bool doesMutates() const { return true; } - virtual void exec(DatabaseImpl* db); + virtual void exec( DatabaseImpl* db ); signals: void indexUpdated(); -public slots: - private: + void indexTable( DatabaseImpl* db, const QString& table ); + QString table; - int pkey; - }; #endif // DATABASECOMMAND_UPDATESEARCHINDEX_H diff --git a/src/libtomahawk/database/databaseimpl.cpp b/src/libtomahawk/database/databaseimpl.cpp index e23b9f9db..59c74fabf 100644 --- a/src/libtomahawk/database/databaseimpl.cpp +++ b/src/libtomahawk/database/databaseimpl.cpp @@ -16,7 +16,7 @@ */ #include "schema.sql.h" -#define CURRENT_SCHEMA_VERSION 16 +#define CURRENT_SCHEMA_VERSION 18 DatabaseImpl::DatabaseImpl( const QString& dbname, Database* parent ) @@ -24,9 +24,8 @@ DatabaseImpl::DatabaseImpl( const QString& dbname, Database* parent ) , m_lastartid( 0 ) , m_lastalbid( 0 ) , m_lasttrkid( 0 ) - , m_fuzzyIndex( *this ) { - connect( this, SIGNAL(indexReady()), parent, SIGNAL(indexReady()) ); + connect( this, SIGNAL( indexReady() ), parent, SIGNAL( indexReady() ) ); db = QSqlDatabase::addDatabase( "QSQLITE", "tomahawk" ); db.setDatabaseName( dbname ); @@ -96,6 +95,8 @@ DatabaseImpl::DatabaseImpl( const QString& dbname, Database* parent ) // in case of unclean shutdown last time: query.exec( "UPDATE source SET isonline = 'false'" ); + + m_fuzzyIndex = new FuzzyIndex( *this ); } @@ -103,6 +104,8 @@ DatabaseImpl::~DatabaseImpl() { m_indexThread.quit(); m_indexThread.wait( 5000 ); + + delete m_fuzzyIndex; } @@ -110,17 +113,17 @@ void DatabaseImpl::loadIndex() { // load ngram index in the background - m_fuzzyIndex.moveToThread( &m_indexThread ); - connect( &m_indexThread, SIGNAL(started()), &m_fuzzyIndex, SLOT(loadNgramIndex()) ); - connect( &m_fuzzyIndex, SIGNAL(indexReady()), this, SIGNAL(indexReady()) ); + m_fuzzyIndex->moveToThread( &m_indexThread ); + connect( &m_indexThread, SIGNAL( started() ), m_fuzzyIndex, SLOT( loadLuceneIndex() ) ); + connect( m_fuzzyIndex, SIGNAL( indexReady() ), this, SIGNAL( indexReady() ) ); m_indexThread.start(); } void -DatabaseImpl::updateSearchIndex( const QString& table, int pkey ) +DatabaseImpl::updateSearchIndex() { - DatabaseCommand* cmd = new DatabaseCommand_UpdateSearchIndex(table, pkey); + DatabaseCommand* cmd = new DatabaseCommand_UpdateSearchIndex(); Database::instance()->enqueue( QSharedPointer( cmd ) ); } @@ -324,41 +327,24 @@ DatabaseImpl::albumId( int artistid, const QString& name_orig, bool& isnew ) QList< int > -DatabaseImpl::searchTable( const QString& table, const QString& name_orig, uint limit ) +DatabaseImpl::searchTable( const QString& table, const QString& name, uint limit ) { QList< int > results; if( table != "artist" && table != "track" && table != "album" ) return results; - QString name = sortname( name_orig ); + QMap< int, float > resultsmap = m_fuzzyIndex->search( table, name ); - // first check for exact matches: - TomahawkSqlQuery query = newquery(); - query.prepare( QString( "SELECT id FROM %1 WHERE sortname = ?" ).arg( table ) ); - query.addBindValue( name ); - query.exec(); - - while( query.next() ) - results.append( query.value( 0 ).toInt() ); - - // ngram stuff only works on tracks over a certain length atm: - if( name_orig.length() > 3 ) + QList< QPair > resultslist; + foreach( int i, resultsmap.keys() ) { - // consult ngram index to find candidates: - QMap< int, float > resultsmap = m_fuzzyIndex.search( table, name ); + resultslist << QPair( i, (float)resultsmap.value( i ) ); + } + qSort( resultslist.begin(), resultslist.end(), DatabaseImpl::scorepairSorter ); - //qDebug() << "results map for" << table << resultsmap.size(); - QList< QPair > resultslist; - foreach( int i, resultsmap.keys() ) - { - resultslist << QPair( i, (float)resultsmap.value( i ) ); - } - qSort( resultslist.begin(), resultslist.end(), DatabaseImpl::scorepairSorter ); - - for( int k = 0; k < resultslist.size() && k < (int)limit; ++k ) - { - results << resultslist.at( k ).first; - } + for( int k = 0; k < resultslist.count(); k++ ) + { + results << resultslist.at( k ).first; } return results; @@ -383,35 +369,6 @@ DatabaseImpl::getTrackFids( int tid ) } -QMap< QString, int > -DatabaseImpl::ngrams( const QString& str_orig ) -{ - static QMap< QString, QMap > memo; - if( memo.contains( str_orig ) ) - return memo.value( str_orig ); - - int n = 3; - QMap ret; - QString str( " " + DatabaseImpl::sortname( str_orig ) + " " ); - int num = str.length(); - QString ngram; - - for( int j = 0; j < num - ( n - 1 ); j++ ) - { - ngram = str.mid( j, n ); - Q_ASSERT( ngram.length() == n ); - - if( ret.contains( ngram ) ) - ret[ngram]++; - else - ret[ngram] = 1; - } - - memo.insert( str_orig, ret ); - return ret; -} - - QString DatabaseImpl::sortname( const QString& str ) { diff --git a/src/libtomahawk/database/databaseimpl.h b/src/libtomahawk/database/databaseimpl.h index 32ec94ad4..7fcd3f299 100644 --- a/src/libtomahawk/database/databaseimpl.h +++ b/src/libtomahawk/database/databaseimpl.h @@ -36,10 +36,9 @@ public: int trackId( int artistid, const QString& name_orig, bool& isnew ); int albumId( int artistid, const QString& name_orig, bool& isnew ); - QList< int > searchTable( const QString& table, const QString& name_orig, uint limit = 10 ); + QList< int > searchTable( const QString& table, const QString& name, uint limit = 10 ); QList< int > getTrackFids( int tid ); - static QMap ngrams( const QString& str_orig ); static QString sortname( const QString& str ); QVariantMap artist( int id ); @@ -54,7 +53,7 @@ public: } // indexes entries from "table" where id >= pkey - void updateSearchIndex( const QString& table, int pkey ); + void updateSearchIndex(); const QString& dbid() const { return m_dbid; } @@ -76,7 +75,7 @@ private: QString m_dbid; QThread m_indexThread; - FuzzyIndex m_fuzzyIndex; + FuzzyIndex* m_fuzzyIndex; }; #endif // DATABASEIMPL_H diff --git a/src/libtomahawk/database/databaseresolver.cpp b/src/libtomahawk/database/databaseresolver.cpp index 60326949f..639a3fede 100644 --- a/src/libtomahawk/database/databaseresolver.cpp +++ b/src/libtomahawk/database/databaseresolver.cpp @@ -4,9 +4,8 @@ #include "database/database.h" #include "database/databasecommand_resolve.h" -DatabaseResolver::DatabaseResolver( bool searchlocal, int weight ) +DatabaseResolver::DatabaseResolver( int weight ) : Resolver() - , m_searchlocal( searchlocal ) , m_weight( weight ) { } @@ -17,13 +16,7 @@ DatabaseResolver::resolve( const QVariant& v ) { //qDebug() << Q_FUNC_INFO << v; - if( !m_searchlocal ) - { - if( Servent::instance()->numConnectedPeers() == 0 ) - return; - } - - DatabaseCommand_Resolve* cmd = new DatabaseCommand_Resolve( v, m_searchlocal ); + DatabaseCommand_Resolve* cmd = new DatabaseCommand_Resolve( v ); connect( cmd, SIGNAL( results( Tomahawk::QID, QList< Tomahawk::result_ptr> ) ), SLOT( gotResults( Tomahawk::QID, QList< Tomahawk::result_ptr> ) ), Qt::QueuedConnection ); @@ -45,5 +38,5 @@ DatabaseResolver::gotResults( const Tomahawk::QID qid, QList< Tomahawk::result_p QString DatabaseResolver::name() const { - return QString( "Database (%1)" ).arg( m_searchlocal ? "local" : "remote" ); + return QString( "DatabaseResolver" ); } diff --git a/src/libtomahawk/database/databaseresolver.h b/src/libtomahawk/database/databaseresolver.h index b40f2a200..81bfca9d4 100644 --- a/src/libtomahawk/database/databaseresolver.h +++ b/src/libtomahawk/database/databaseresolver.h @@ -13,7 +13,7 @@ class DLLEXPORT DatabaseResolver : public Tomahawk::Resolver Q_OBJECT public: - explicit DatabaseResolver( bool searchlocal, int weight ); + explicit DatabaseResolver( int weight ); virtual QString name() const; virtual unsigned int weight() const { return m_weight; } @@ -26,7 +26,6 @@ private slots: void gotResults( const Tomahawk::QID qid, QList< Tomahawk::result_ptr> results ); private: - bool m_searchlocal; int m_weight; }; diff --git a/src/libtomahawk/database/databaseworker.cpp b/src/libtomahawk/database/databaseworker.cpp index e626ccb6d..374590ada 100644 --- a/src/libtomahawk/database/databaseworker.cpp +++ b/src/libtomahawk/database/databaseworker.cpp @@ -40,9 +40,10 @@ DatabaseWorker::run() void DatabaseWorker::enqueue( const QSharedPointer& cmd ) { + m_outstanding++; + QMutexLocker lock( &m_mut ); m_commands << cmd; - m_outstanding++; if ( m_outstanding == 1 ) QTimer::singleShot( 0, this, SLOT( doWork() ) ); @@ -163,11 +164,7 @@ DatabaseWorker::doWork() cmd->emitFinished(); - { - QMutexLocker lock( &m_mut ); - m_outstanding--; - } - + m_outstanding--; if ( m_outstanding > 0 ) QTimer::singleShot( 0, this, SLOT( doWork() ) ); } diff --git a/src/libtomahawk/database/fuzzyindex.cpp b/src/libtomahawk/database/fuzzyindex.cpp index a4dbf5478..67906a9fb 100644 --- a/src/libtomahawk/database/fuzzyindex.cpp +++ b/src/libtomahawk/database/fuzzyindex.cpp @@ -1,139 +1,146 @@ #include "fuzzyindex.h" #include "databaseimpl.h" +#include "utils/tomahawkutils.h" +#include #include +#include -FuzzyIndex::FuzzyIndex( DatabaseImpl &db ) + +FuzzyIndex::FuzzyIndex( DatabaseImpl& db ) : QObject() , m_db( db ) - , m_loaded( false ) + , m_luceneReader( 0 ) + , m_luceneSearcher( 0 ) { + QString lucenePath = TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" ); + bool create = !lucene::index::IndexReader::indexExists( lucenePath.toStdString().c_str() ); + m_luceneDir = lucene::store::FSDirectory::getDirectory( lucenePath.toStdString().c_str(), create ); + + m_analyzer = new lucene::analysis::SimpleAnalyzer(); +} + + +FuzzyIndex::~FuzzyIndex() +{ + delete m_luceneSearcher; + delete m_luceneReader; + delete m_analyzer; + delete m_luceneDir; } void -FuzzyIndex::loadNgramIndex() +FuzzyIndex::beginIndexing() { - // this updates the index in the DB, if needed: - qDebug() << "Checking catalogue is fully indexed..."; - m_db.updateSearchIndex( "artist", 0 ); - m_db.updateSearchIndex( "album", 0 ); - m_db.updateSearchIndex( "track", 0 ); + lucene::index::IndexWriter luceneWriter = lucene::index::IndexWriter( m_luceneDir, m_analyzer, true ); + m_mutex.lock(); +} - // loads index from DB into memory: - qDebug() << "Loading search index for catalogue metadata..." << thread(); - loadNgramIndex_helper( m_artist_ngrams, "artist" ); - loadNgramIndex_helper( m_album_ngrams, "album" ); - loadNgramIndex_helper( m_track_ngrams, "track" ); - m_loaded = true; + +void +FuzzyIndex::endIndexing() +{ + m_mutex.unlock(); +} + + +void +FuzzyIndex::appendFields( const QString& table, const QMap< unsigned int, QString >& fields ) +{ + delete m_luceneSearcher; + delete m_luceneReader; + m_luceneSearcher = 0; + m_luceneReader = 0; + + bool create = !lucene::index::IndexReader::indexExists( TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" ).toStdString().c_str() ); + lucene::index::IndexWriter luceneWriter = lucene::index::IndexWriter( m_luceneDir, m_analyzer, create ); + lucene::document::Document doc; + + QMapIterator< unsigned int, QString > it( fields ); + while ( it.hasNext() ) + { + it.next(); + unsigned int id = it.key(); + QString name = it.value(); + + { + lucene::document::Field* field = new lucene::document::Field( table.toStdWString().c_str(), name.toStdWString().c_str(), + lucene::document::Field::STORE_YES | lucene::document::Field::INDEX_UNTOKENIZED ); + doc.add( *field ); + } + + { + lucene::document::Field* field = new lucene::document::Field( _T( "id" ), QString::number( id ).toStdWString().c_str(), + lucene::document::Field::STORE_YES | lucene::document::Field::INDEX_NO ); + doc.add( *field ); + } + + luceneWriter.addDocument( &doc ); + doc.clear(); + } + + luceneWriter.close(); +} + + +void +FuzzyIndex::loadLuceneIndex() +{ emit indexReady(); } -void -FuzzyIndex::loadNgramIndex_helper( QHash< QString, QMap >& idx, const QString& table, unsigned int fromkey ) -{ - QTime t; - t.start(); - - TomahawkSqlQuery query = m_db.newquery(); - query.exec( QString( "SELECT ngram, id, num " - "FROM %1_search_index " - "WHERE id >= %2 " - "ORDER BY ngram" ).arg( table ).arg( fromkey ) ); - - QMap ngram_idx; - QString lastngram; - while( query.next() ) - { - const QString ng = query.value( 0 ).toString(); - if( lastngram.isEmpty() ) - lastngram = ng; - - if( ng != lastngram ) - { - idx.insert( lastngram, ngram_idx ); - lastngram = ng; - ngram_idx.clear(); - } - - ngram_idx.insert( query.value( 1 ).toUInt(), - query.value( 2 ).toUInt() ); - } - - idx.insert( lastngram, ngram_idx ); - qDebug() << "Loaded" << idx.size() - << "ngram entries for" << table - << "in" << t.elapsed() << "ms"; -} - - -void FuzzyIndex::mergeIndex( const QString& table, const QHash< QString, QMap >& tomerge ) -{ - qDebug() << Q_FUNC_INFO << table << tomerge.keys().size(); - - QHash< QString, QMap >* idx; - if ( table == "artist" ) idx = &m_artist_ngrams; - else if( table == "album" ) idx = &m_album_ngrams; - else if( table == "track" ) idx = &m_track_ngrams; - else Q_ASSERT( false ); - - if( tomerge.size() == 0 ) - return; - - if( idx->size() == 0 ) - { - *idx = tomerge; - } - else - { - QList tmk = tomerge.keys(); - foreach( const QString& ngram, tmk ) - { - if( idx->contains( ngram ) ) - { - QList tmkn = tomerge[ngram].keys(); - foreach( quint32 id, tmkn ) - { - (*idx)[ ngram ][ id ] += tomerge[ngram][id]; - } - } - else - { - idx->insert( ngram, tomerge[ngram] ); - } - } - } - - qDebug() << Q_FUNC_INFO << table << "merge complete, num items:" << tomerge.size(); -} - - QMap< int, float > FuzzyIndex::search( const QString& table, const QString& name ) { + QMutexLocker lock( &m_mutex ); + QMap< int, float > resultsmap; - - QHash< QString, QMap >* idx; - if( table == "artist" ) idx = &m_artist_ngrams; - else if( table == "album" ) idx = &m_album_ngrams; - else if( table == "track" ) idx = &m_track_ngrams; - - QMap ngramsmap = DatabaseImpl::ngrams( name ); - foreach( const QString& ngram, ngramsmap.keys() ) + if ( !m_luceneReader ) { - if( !idx->contains( ngram ) ) - continue; - - //qDebug() << name_orig << "NGRAM:" << ngram << "candidates:" << (*idx)[ngram].size(); - QMapIterator iter( (*idx)[ngram] ); - while( iter.hasNext() ) + if ( !lucene::index::IndexReader::indexExists( TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" ).toStdString().c_str() ) ) { - iter.next(); - resultsmap[ (int) iter.key() ] += (float) iter.value(); + qDebug() << Q_FUNC_INFO << "index didn't exist."; + return resultsmap; + } + + m_luceneReader = lucene::index::IndexReader::open( m_luceneDir ); + m_luceneSearcher = new lucene::search::IndexSearcher( m_luceneReader ); + } + + if ( name.isEmpty() ) + return resultsmap; + + lucene::analysis::SimpleAnalyzer analyzer; + lucene::queryParser::QueryParser parser( table.toStdWString().c_str(), m_analyzer ); + lucene::search::Hits* hits = 0; + + lucene::search::FuzzyQuery* qry = new lucene::search::FuzzyQuery( new lucene::index::Term( table.toStdWString().c_str(), + name.toStdWString().c_str() ) ); + hits = m_luceneSearcher->search( qry ); + + for ( unsigned int i = 0; i < hits->length(); i++ ) + { + lucene::document::Document* d = &hits->doc( i ); + + float score = hits->score( i ); + int id = QString::fromWCharArray( d->get( _T( "id" ) ) ).toInt(); + QString result = QString::fromWCharArray( d->get( table.toStdWString().c_str() ) ); + + if ( result.toLower() == name.toLower() ) + score = 1.0; + else + score = qMin( score, (float)0.99 ); + + if ( score > 0.05 ) + { + resultsmap.insert( id, score ); +// qDebug() << "Hitres:" << result << id << score << table << name; } } + return resultsmap; } diff --git a/src/libtomahawk/database/fuzzyindex.h b/src/libtomahawk/database/fuzzyindex.h index 0258493eb..df49ba31b 100644 --- a/src/libtomahawk/database/fuzzyindex.h +++ b/src/libtomahawk/database/fuzzyindex.h @@ -5,33 +5,59 @@ #include #include #include +#include + +namespace lucene +{ + namespace analysis + { + class SimpleAnalyzer; + } + namespace store + { + class Directory; + } + namespace index + { + class IndexReader; + class IndexWriter; + } + namespace search + { + class IndexSearcher; + } +} class DatabaseImpl; - class FuzzyIndex : public QObject { Q_OBJECT public: - explicit FuzzyIndex( DatabaseImpl &db ); + explicit FuzzyIndex( DatabaseImpl& db ); + ~FuzzyIndex(); + void beginIndexing(); + void endIndexing(); + void appendFields( const QString& table, const QMap< unsigned int, QString >& fields ); + signals: void indexReady(); public slots: - void loadNgramIndex(); + void loadLuceneIndex(); + QMap< int, float > search( const QString& table, const QString& name ); - void mergeIndex( const QString& table, const QHash< QString, QMap >& tomerge ); private: - void loadNgramIndex_helper( QHash< QString, QMap >& idx, const QString& table, unsigned int fromkey = 0 ); + DatabaseImpl& m_db; + QMutex m_mutex; - // maps an ngram to {track id, num occurences} - QHash< QString, QMap > m_artist_ngrams, m_album_ngrams, m_track_ngrams; - - DatabaseImpl & m_db; - bool m_loaded; + lucene::analysis::SimpleAnalyzer* m_analyzer; + lucene::store::Directory* m_luceneDir; + lucene::index::IndexReader* m_luceneReader; + lucene::search::IndexSearcher* m_luceneSearcher; }; #endif // FUZZYINDEX_H diff --git a/src/libtomahawk/database/gen_schema.h.sh b/src/libtomahawk/database/gen_schema.h.sh index b8a6a6b79..424ee18c0 100755 --- a/src/libtomahawk/database/gen_schema.h.sh +++ b/src/libtomahawk/database/gen_schema.h.sh @@ -1,4 +1,8 @@ #!/bin/bash + +# !!!! You need to manually generate schema.sql.h when the schema changes: +# ./gen_schema.h.sh ./schema.sql tomahawk > schema.sql.h + schema=$1 name=$2 diff --git a/src/libtomahawk/database/schema.sql b/src/libtomahawk/database/schema.sql index 460a34f59..7e806f5f2 100644 --- a/src/libtomahawk/database/schema.sql +++ b/src/libtomahawk/database/schema.sql @@ -14,6 +14,8 @@ CREATE TABLE IF NOT EXISTS oplog ( CREATE UNIQUE INDEX oplog_guid ON oplog(guid); CREATE INDEX oplog_source ON oplog(source); + + -- the basic 3 catalogue tables: CREATE TABLE IF NOT EXISTS artist ( @@ -39,6 +41,8 @@ CREATE TABLE IF NOT EXISTS album ( ); CREATE UNIQUE INDEX album_artist_sortname ON album(artist,sortname); + + -- Source, typically a remote peer. CREATE TABLE IF NOT EXISTS source ( @@ -51,6 +55,7 @@ CREATE TABLE IF NOT EXISTS source ( CREATE UNIQUE INDEX source_name ON source(name); + -- playlists CREATE TABLE IF NOT EXISTS playlist ( @@ -64,9 +69,6 @@ CREATE TABLE IF NOT EXISTS playlist ( currentrevision TEXT REFERENCES playlist_revision(guid) DEFERRABLE INITIALLY DEFERRED ); ---INSERT INTO playlist(guid, title, info, currentrevision) --- VALUES('playlistguid-1','Test Playlist','this playlist automatically created and used for testing','revisionguid-1'); - CREATE TABLE IF NOT EXISTS playlist_item ( guid TEXT PRIMARY KEY, playlist TEXT NOT NULL REFERENCES playlist(guid) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED, @@ -79,16 +81,8 @@ CREATE TABLE IF NOT EXISTS playlist_item ( addedby INTEGER REFERENCES source(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED, -- who added this to the playlist result_hint TEXT -- hint as to a result, to avoid using the resolver ); - CREATE INDEX playlist_item_playlist ON playlist_item(playlist); ---INSERT INTO playlist_item(guid, playlist, trackname, artistname) --- VALUES('itemguid-1','playlistguid-1','track name 01','artist name 01'); ---INSERT INTO playlist_item(guid, playlist, trackname, artistname) --- VALUES('itemguid-2','playlistguid-1','track name 02','artist name 02'); ---INSERT INTO playlist_item(guid, playlist, trackname, artistname) --- VALUES('itemguid-3','playlistguid-1','track name 03','artist name 03'); --- CREATE TABLE IF NOT EXISTS playlist_revision ( guid TEXT PRIMARY KEY, playlist TEXT NOT NULL REFERENCES playlist(guid) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED, @@ -99,31 +93,6 @@ CREATE TABLE IF NOT EXISTS playlist_revision ( ); --- the trigram search indexes - -CREATE TABLE IF NOT EXISTS artist_search_index ( - ngram TEXT NOT NULL, - id INTEGER NOT NULL REFERENCES artist(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED, - num INTEGER NOT NULL DEFAULT 1, - PRIMARY KEY(ngram, id) -); --- CREATE INDEX artist_search_index_ngram ON artist_search_index(ngram); - -CREATE TABLE IF NOT EXISTS album_search_index ( - ngram TEXT NOT NULL, - id INTEGER NOT NULL REFERENCES album(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED, - num INTEGER NOT NULL DEFAULT 1, - PRIMARY KEY(ngram, id) -); --- CREATE INDEX album_search_index_ngram ON album_search_index(ngram); - -CREATE TABLE IF NOT EXISTS track_search_index ( - ngram TEXT NOT NULL, - id INTEGER NOT NULL REFERENCES track(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED, - num INTEGER NOT NULL DEFAULT 1, - PRIMARY KEY(ngram, id) -); --- CREATE INDEX track_search_index_ngram ON track_search_index(ngram); -- files on disk and joinage with catalogue. physical properties of files only: @@ -149,7 +118,6 @@ CREATE TABLE IF NOT EXISTS dirs_scanned ( mtime INTEGER NOT NULL ); - CREATE TABLE IF NOT EXISTS file_join ( file INTEGER PRIMARY KEY REFERENCES file(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED, artist INTEGER NOT NULL REFERENCES artist(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED, @@ -164,9 +132,9 @@ CREATE INDEX file_join_album ON file_join(album); -- tags, weighted and by source (rock, jazz etc) + -- weight is always 1.0 if tag provided by our user. -- may be less from aggregate sources like lastfm global tags - CREATE TABLE IF NOT EXISTS track_tags ( id INTEGER PRIMARY KEY, -- track id source INTEGER REFERENCES source(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED, @@ -209,6 +177,7 @@ CREATE INDEX track_attrib_id ON track_attributes(id); CREATE INDEX track_attrib_k ON track_attributes(k); + -- playback history -- if source=null, file is local to this machine @@ -223,10 +192,11 @@ CREATE INDEX playback_log_source ON playback_log(source); CREATE INDEX playback_log_track ON playback_log(track); + -- Schema version, and misc tomahawk settings relating to the collection db CREATE TABLE IF NOT EXISTS settings ( k TEXT NOT NULL PRIMARY KEY, v TEXT NOT NULL DEFAULT '' ); -INSERT INTO settings(k,v) VALUES('schema_version', '16'); +INSERT INTO settings(k,v) VALUES('schema_version', '18'); diff --git a/src/libtomahawk/database/schema.sql.h b/src/libtomahawk/database/schema.sql.h index 8b504ce5d..77a472959 100644 --- a/src/libtomahawk/database/schema.sql.h +++ b/src/libtomahawk/database/schema.sql.h @@ -1,5 +1,5 @@ /* - This file was automatically generated from schema.sql on Thu Jan 6 09:52:57 CET 2011. + This file was automatically generated from ./schema.sql on Sat Jan 29 10:28:18 CET 2011. */ static const char * tomahawk_schema_sql = @@ -73,24 +73,6 @@ static const char * tomahawk_schema_sql = " timestamp INTEGER NOT NULL DEFAULT 0," " previous_revision TEXT REFERENCES playlist_revision(guid) DEFERRABLE INITIALLY DEFERRED" ");" -"CREATE TABLE IF NOT EXISTS artist_search_index (" -" ngram TEXT NOT NULL," -" id INTEGER NOT NULL REFERENCES artist(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED," -" num INTEGER NOT NULL DEFAULT 1," -" PRIMARY KEY(ngram, id)" -");" -"CREATE TABLE IF NOT EXISTS album_search_index (" -" ngram TEXT NOT NULL," -" id INTEGER NOT NULL REFERENCES album(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED," -" num INTEGER NOT NULL DEFAULT 1," -" PRIMARY KEY(ngram, id)" -");" -"CREATE TABLE IF NOT EXISTS track_search_index (" -" ngram TEXT NOT NULL," -" id INTEGER NOT NULL REFERENCES track(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED," -" num INTEGER NOT NULL DEFAULT 1," -" PRIMARY KEY(ngram, id)" -");" "CREATE TABLE IF NOT EXISTS file (" " id INTEGER PRIMARY KEY AUTOINCREMENT," " source INTEGER REFERENCES source(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED," @@ -162,7 +144,7 @@ static const char * tomahawk_schema_sql = " k TEXT NOT NULL PRIMARY KEY," " v TEXT NOT NULL DEFAULT ''" ");" -"INSERT INTO settings(k,v) VALUES('schema_version', '16');" +"INSERT INTO settings(k,v) VALUES('schema_version', '18');" ; const char * get_tomahawk_sql() diff --git a/src/libtomahawk/pipeline.cpp b/src/libtomahawk/pipeline.cpp index 158ceb203..d0c75db0b 100644 --- a/src/libtomahawk/pipeline.cpp +++ b/src/libtomahawk/pipeline.cpp @@ -29,7 +29,7 @@ Pipeline::Pipeline( QObject* parent ) void Pipeline::databaseReady() { - connect( Database::instance(), SIGNAL(indexReady()), this, SLOT(indexReady()), Qt::QueuedConnection ); + connect( Database::instance(), SIGNAL( indexReady() ), this, SLOT( indexReady() ), Qt::QueuedConnection ); Database::instance()->loadIndex(); } @@ -73,7 +73,7 @@ Pipeline::addResolver( Resolver* r, bool sort ) void -Pipeline::add( const QList& qlist, bool prioritized ) +Pipeline::resolve( const QList& qlist, bool prioritized ) { { QMutexLocker lock( &m_mut ); @@ -103,12 +103,21 @@ Pipeline::add( const QList& qlist, bool prioritized ) void -Pipeline::add( const query_ptr& q, bool prioritized ) +Pipeline::resolve( const query_ptr& q, bool prioritized ) { - //qDebug() << Q_FUNC_INFO << (qlonglong)q.data() << q->toString(); + if ( q.isNull() ) + return; + QList< query_ptr > qlist; qlist << q; - add( qlist, prioritized ); + resolve( qlist, prioritized ); +} + + +void +Pipeline::resolve( QID qid, bool prioritized ) +{ + resolve( query( qid ), prioritized ); } @@ -117,12 +126,22 @@ Pipeline::reportResults( QID qid, const QList< result_ptr >& results ) { QMutexLocker lock( &m_mut ); - if( !m_qids.contains( qid ) ) + if ( !m_qids.contains( qid ) ) { qDebug() << "reportResults called for unknown QID"; return; } + unsigned int state = m_qidsState.value( qid ) - 1; + m_qidsState.insert( qid, state ); + + if ( state == 0 ) + { + // All resolvers have reported back their results for this query now + const query_ptr& q = m_qids.value( qid ); + q->onResolvingFinished(); + } + if ( !results.isEmpty() ) { //qDebug() << Q_FUNC_INFO << qid; @@ -189,6 +208,9 @@ Pipeline::shunt( const query_ptr& q ) // resolvers aren't allowed to block in this call: //qDebug() << "Dispaching to resolver" << r->name(); + + unsigned int state = m_qidsState.value( q->id() ); + m_qidsState.insert( q->id(), state + 1 ); r->resolve( q->toVariant() ); } else diff --git a/src/libtomahawk/pipeline.h b/src/libtomahawk/pipeline.h index 873a91efa..615d6d1d5 100644 --- a/src/libtomahawk/pipeline.h +++ b/src/libtomahawk/pipeline.h @@ -27,9 +27,6 @@ public: explicit Pipeline( QObject* parent = 0 ); -// const query_ptr& query( QID qid ) const; -// result_ptr result( RID rid ) const; - void reportResults( QID qid, const QList< result_ptr >& results ); /// sorter to rank resolver priority @@ -49,8 +46,9 @@ public: } public slots: - void add( const query_ptr& q, bool prioritized = true ); - void add( const QList& qlist, bool prioritized = true ); + void resolve( const query_ptr& q, bool prioritized = true ); + void resolve( const QList& qlist, bool prioritized = true ); + void resolve( QID qid, bool prioritized = true ); void databaseReady(); signals: @@ -64,6 +62,8 @@ private slots: private: QList< Resolver* > m_resolvers; + + QMap< QID, unsigned int > m_qidsState; QMap< QID, query_ptr > m_qids; QMap< RID, result_ptr > m_rids; diff --git a/src/libtomahawk/playlist.cpp b/src/libtomahawk/playlist.cpp index 26fd23cbd..0d6c2d285 100644 --- a/src/libtomahawk/playlist.cpp +++ b/src/libtomahawk/playlist.cpp @@ -20,7 +20,7 @@ using namespace Tomahawk; void PlaylistEntry::setQueryVariant( const QVariant& v ) { - m_query = query_ptr( new Query( v ) ); + m_query = Tomahawk::Query::get( v, false ); } @@ -342,7 +342,8 @@ Playlist::resolve() { qlist << p->query(); } - Pipeline::instance()->add( qlist ); + + Pipeline::instance()->resolve( qlist ); } diff --git a/src/libtomahawk/playlist/collectionmodel.cpp b/src/libtomahawk/playlist/collectionmodel.cpp index d951b4aed..212f60c79 100644 --- a/src/libtomahawk/playlist/collectionmodel.cpp +++ b/src/libtomahawk/playlist/collectionmodel.cpp @@ -195,7 +195,7 @@ CollectionModel::onTracksAdded( const QList& tracks, const collection_ PlItem* plitem; foreach( const QVariant& v, tracks ) { - Tomahawk::query_ptr query = query_ptr( new Query( v ) ); + Tomahawk::query_ptr query = Tomahawk::Query::get( v, false ); // FIXME: needs merging // Manually add a result, since it's coming from the local collection diff --git a/src/libtomahawk/query.cpp b/src/libtomahawk/query.cpp index 5a9d5ceea..57ae534d6 100644 --- a/src/libtomahawk/query.cpp +++ b/src/libtomahawk/query.cpp @@ -2,14 +2,28 @@ #include +#include "database/database.h" +#include "pipeline.h" +#include "sourcelist.h" + using namespace Tomahawk; +query_ptr +Query::get( const QVariant& v, bool autoResolve ) +{ + query_ptr q = query_ptr( new Query( v ) ); + + if ( autoResolve ) + Pipeline::instance()->resolve( q ); + return q; +} + + Query::Query( const QVariant& v ) : m_v( v ) , m_solved( false ) { - // ensure a QID is present: QVariantMap m = m_v.toMap(); m_artist = m.value( "artist" ).toString(); @@ -17,6 +31,9 @@ Query::Query( const QVariant& v ) m_track = m.value( "track" ).toString(); m_qid = m.value( "qid" ).toString(); + + connect( SourceList::instance(), SIGNAL( sourceAdded( Tomahawk::source_ptr ) ), SLOT( refreshResults() ), Qt::QueuedConnection ); + connect( Database::instance(), SIGNAL( indexReady() ), SLOT( refreshResults() ), Qt::QueuedConnection ); } @@ -46,6 +63,13 @@ Query::addResults( const QList< Tomahawk::result_ptr >& newresults ) } +void +Query::refreshResults() +{ + Pipeline::instance()->resolve( id() ); +} + + void Query::resultUnavailable() { @@ -83,6 +107,14 @@ Query::removeResult( const Tomahawk::result_ptr& result ) } +void +Query::onResolvingFinished() +{ +// qDebug() << Q_FUNC_INFO << "Finished resolving." << toString(); + emit resolvingFinished( !m_results.isEmpty() ); +} + + QList< result_ptr > Query::results() const { diff --git a/src/libtomahawk/query.h b/src/libtomahawk/query.h index 2270396f7..0a17e2801 100644 --- a/src/libtomahawk/query.h +++ b/src/libtomahawk/query.h @@ -20,6 +20,7 @@ class DLLEXPORT Query : public QObject Q_OBJECT public: + static query_ptr get( const QVariant& v, bool autoResolve = true ); explicit Query( const QVariant& v ); QVariant toVariant() const { return m_v; } @@ -39,7 +40,7 @@ public: bool solved() const { return m_solved; } unsigned int lastPipelineWeight() const { return m_lastpipelineweight; } - void setLastPipelineWeight( unsigned int w ) { m_lastpipelineweight = w;} + void setLastPipelineWeight( unsigned int w ) { m_lastpipelineweight = w; } /// for debug output: QString toString() const @@ -54,15 +55,20 @@ public: signals: void resultsAdded( const QList& ); void resultsRemoved( const Tomahawk::result_ptr& ); - void solvedStateChanged( bool state ); + void solvedStateChanged( bool state ); + void resolvingFinished( bool hasResults ); + public slots: /// (indirectly) called by resolver plugins when results are found void addResults( const QList< Tomahawk::result_ptr >& ); void removeResult( const Tomahawk::result_ptr& ); + void onResolvingFinished(); + private slots: void resultUnavailable(); + void refreshResults(); private: mutable QMutex m_mut; diff --git a/src/libtomahawk/result.cpp b/src/libtomahawk/result.cpp index f16eb0a22..4888609bb 100644 --- a/src/libtomahawk/result.cpp +++ b/src/libtomahawk/result.cpp @@ -59,7 +59,7 @@ Result::toString() const Tomahawk::query_ptr Result::toQuery() const { - Tomahawk::query_ptr query = Tomahawk::query_ptr( new Tomahawk::Query( toVariant() ) ); + Tomahawk::query_ptr query = Tomahawk::Query::get( toVariant(), false ); return query; } diff --git a/src/libtomahawk/utils/xspfloader.cpp b/src/libtomahawk/utils/xspfloader.cpp index 2397ee648..f8125c945 100644 --- a/src/libtomahawk/utils/xspfloader.cpp +++ b/src/libtomahawk/utils/xspfloader.cpp @@ -105,7 +105,7 @@ XSPFLoader::gotBody() v.insert( "album", e.firstChildElement( "album" ).text() ); v.insert( "track", e.firstChildElement( "title" ).text() ); - p->setQuery( Tomahawk::query_ptr(new Tomahawk::Query(v)) ); + p->setQuery( Tomahawk::Query::get( v ) ); m_entries << p; } diff --git a/src/libtomahawk/widgets/newplaylistwidget.cpp b/src/libtomahawk/widgets/newplaylistwidget.cpp index 75c5d065f..7882532c0 100644 --- a/src/libtomahawk/widgets/newplaylistwidget.cpp +++ b/src/libtomahawk/widgets/newplaylistwidget.cpp @@ -11,7 +11,6 @@ #include "widgets/overlaywidget.h" -#include "pipeline.h" #include "utils/xspfloader.h" #include "sourcelist.h" @@ -115,8 +114,6 @@ NewPlaylistWidget::suggestionsFound() ql.append( entry->query() ); } - Tomahawk::Pipeline::instance()->add( ql ); - loader->deleteLater(); } diff --git a/src/tomahawkapp.cpp b/src/tomahawkapp.cpp index ed0f1e2e1..69236040e 100644 --- a/src/tomahawkapp.cpp +++ b/src/tomahawkapp.cpp @@ -306,8 +306,7 @@ void TomahawkApp::setupPipeline() { // setup resolvers for local content, and (cached) remote collection content - Pipeline::instance()->addResolver( new DatabaseResolver( true, 100 ) ); - Pipeline::instance()->addResolver( new DatabaseResolver( false, 90 ) ); + Pipeline::instance()->addResolver( new DatabaseResolver( 100 ) ); // new ScriptResolver("/home/rj/src/tomahawk-core/contrib/magnatune/magnatune-resolver.php"); } diff --git a/src/tomahawkwindow.cpp b/src/tomahawkwindow.cpp index f6f0272e0..d4d2b5b30 100644 --- a/src/tomahawkwindow.cpp +++ b/src/tomahawkwindow.cpp @@ -351,5 +351,5 @@ TomahawkWindow::setWindowTitle( const QString& title ) void TomahawkWindow::showAboutTomahawk() { - QMessageBox::about( this, "About Tomahawk", "Copyright 2010 Christian Muehlhaeuser \n\nThanks to: Leo Franchi, Jeff Mitchell, Dominik Schmidt, Alejandro Wainzinger and Steve Robertson" ); + QMessageBox::about( this, "About Tomahawk", "Copyright 2010, 2011 Christian Muehlhaeuser \n\nThanks to: Leo Franchi, Jeff Mitchell, Dominik Schmidt, Alejandro Wainzinger and Steve Robertson" ); } diff --git a/src/web/api_v1.h b/src/web/api_v1.h index 0a77a679d..6d89beddd 100644 --- a/src/web/api_v1.h +++ b/src/web/api_v1.h @@ -112,8 +112,7 @@ public slots: m.insert( "track", event->url.queryItemValue("track") ); m.insert( "qid", qid ); - Tomahawk::query_ptr qry( new Tomahawk::Query( m ) ); - Tomahawk::Pipeline::instance()->add( qry ); + Tomahawk::query_ptr qry = Tomahawk::Query::get( m ); QVariantMap r; r.insert( "qid", qid ); diff --git a/src/xmppbot/xmppbot.cpp b/src/xmppbot/xmppbot.cpp index 0745a07e8..ccb7cc40c 100644 --- a/src/xmppbot/xmppbot.cpp +++ b/src/xmppbot/xmppbot.cpp @@ -5,7 +5,6 @@ #include "album.h" #include "typedefs.h" #include "tomahawksettings.h" -#include "pipeline.h" #include "audio/audioengine.h" @@ -145,11 +144,10 @@ void XMPPBot::handleMessage(const Message& msg, MessageSession* session) QVariantMap qv; qv["artist"] = tokens.first().trimmed(); qv["track"] = tokens.last().trimmed(); - Tomahawk::query_ptr q( new Tomahawk::Query( qv ) ); + Tomahawk::query_ptr q = Tomahawk::Query::get( qv ); connect( q.data(), SIGNAL( resultsAdded( QList ) ), SLOT( onResultsAdded( QList ) ) ); - Tomahawk::Pipeline::instance()->add( q ); return; } else if ( body.startsWith( "stop" ) )