* Tomahawk now relies on Lucene for indexing / searching Artists, Albums and Tracks.

* Lucene index is stored on disk for faster startups.
* Queries are now auto-resolved when retrieved via Query::get( QVariant, true ). The second parameter is optional and true by default.
* Updated database schema and removed the old ngram tables.
* Hopefully didn't break too much :-)

CMakeLists.txt

@@ -16,6 +16,7 @@ ENDIF()
 FIND_PACKAGE( Taglib 1.6.0 REQUIRED )
 FIND_PACKAGE( LibLastFm 0.3.3 REQUIRED )
 FIND_PACKAGE( LibEchonest REQUIRED )
+FIND_PACKAGE( CLucene REQUIRED )
 
 IF( UNIX AND NOT APPLE )
     ADD_SUBDIRECTORY( alsa-playback )

src/libtomahawk/CMakeLists.txt

@@ -228,6 +228,8 @@ set( libUI ${libUI}
 
 include_directories( . ${CMAKE_CURRENT_BINARY_DIR} .. 
     ${QT_INCLUDE_DIR}
+    ${CLUCENE_INCLUDE_DIR}
+    ${CLUCENE_LIBRARY_DIR}
     ../../libportfwd/include
     ../../include
     ../network
@@ -296,6 +298,7 @@ target_link_libraries( tomahawklib
     ogg
     FLAC++
     jdns
+    clucene
 )
 
 install( TARGETS tomahawklib DESTINATION lib )

src/libtomahawk/database/database.h

@@ -16,10 +16,9 @@
 
     HOWEVER, we're using the command pattern to serialize access to the database
     and provide an async api. You create a DatabaseCommand object, and add it to
-    the queue of work. There is a single thread responsible for exec'ing all
+    the queue of work. There is a threadpool responsible for exec'ing all
-    the commands, so sqlite only does one thing at a time.
+    the non-mutating (readonly) commands and one separate thread for mutating ones,
-
+    so sqlite doesn't write to the Database from multiple threads.
-    Update: 1 thread for mutates, one for readonly queries.
 */
 class DLLEXPORT Database : public QObject
 {

src/libtomahawk/database/databasecommand_addfiles.cpp

@@ -16,7 +16,7 @@ QVariantList
 DatabaseCommand_AddFiles::files() const
 {
     QVariantList list;
-    foreach( const QVariant& v, m_files )
+    foreach ( const QVariant& v, m_files )
     {
         // replace url with the id, we don't leak file paths over the network.
         QVariantMap m = v.toMap();
@@ -73,11 +73,7 @@ DatabaseCommand_AddFiles::exec( DatabaseImpl* dbi )
     query_trackattr.prepare( "INSERT INTO track_attributes(id, k, v) "
                              "VALUES (?,?,?)" );
     query_file_del.prepare( QString( "DELETE FROM file WHERE source %1 AND url = ?" )
-                               .arg( source()->isLocal() ? "IS NULL" : QString( "= %1" ).arg( source()->id() )
+                               .arg( source()->isLocal() ? "IS NULL" : QString( "= %1" ).arg( source()->id() ) ) );
-                          ) );
-
-    int maxart, maxalb, maxtrk; // store max id, so we can index new ones after
-    maxart = maxalb = maxtrk = 0;
 
     int added = 0;
     QVariant srcid = source()->isLocal() ?
@@ -126,29 +122,28 @@ DatabaseCommand_AddFiles::exec( DatabaseImpl* dbi )
         }
         else
         {
-            if( added % 100 == 0 ) qDebug() << "Inserted" << added;
+            if( added % 100 == 0 )
+                qDebug() << "Inserted" << added;
         }
         // get internal IDs for art/alb/trk
         fileid = query_file.lastInsertId().toInt();
 
         // insert the new fileid, set the url for our use:
         m.insert( "id", fileid );
-        if( !source()->isLocal() ) m["url"] = QString( "servent://%1\t%2" )
+        if( !source()->isLocal() )
-                                                 .arg( source()->userName() )
+            m["url"] = QString( "servent://%1\t%2" )
-                                                 .arg( fileid );
+                          .arg( source()->userName() )
+                          .arg( fileid );
         v = m;
 
         bool isnew;
         int artid = dbi->artistId( artist, isnew );
         if( artid < 1 ) continue;
-        if( isnew && maxart == 0 ) maxart = artid;
 
         int trkid = dbi->trackId( artid, track, isnew );
         if( trkid < 1 ) continue;
-        if( isnew && maxtrk == 0 ) maxtrk = trkid;
 
         int albid = dbi->albumId( artid, album, isnew );
-        if( albid > 0 && isnew && maxalb == 0 ) maxalb = albid;
 
         // Now add the association
         query_filejoin.bindValue( 0, fileid );
@@ -172,11 +167,8 @@ DatabaseCommand_AddFiles::exec( DatabaseImpl* dbi )
     qDebug() << "Inserted" << added;
 
     // TODO building the index could be a separate job, outside this transaction
-    if(maxart) dbi->updateSearchIndex( "artist", maxart );
+    dbi->updateSearchIndex();
-    if(maxalb) dbi->updateSearchIndex( "album", maxalb );
-    if(maxtrk) dbi->updateSearchIndex( "track", maxtrk );
 
     qDebug() << "Committing" << added << "tracks...";
-    qDebug() << "Done.";
     emit done( m_files, source()->collection() );
 }

src/libtomahawk/database/databasecommand_alltracks.cpp

@@ -89,7 +89,7 @@ DatabaseCommand_AllTracks::exec( DatabaseImpl* dbi )
             attr[ attrQuery.value( 0 ).toString() ] = attrQuery.value( 1 ).toString();
         }
 
-        Tomahawk::query_ptr query = Tomahawk::query_ptr( new Tomahawk::Query( t ) );
+        Tomahawk::query_ptr query = Tomahawk::Query::get( t, false );
         t["score"] = 1.0;
 
         Tomahawk::result_ptr result = Tomahawk::result_ptr( new Tomahawk::Result( t, m_collection ) );

src/libtomahawk/database/databasecommand_loadplaylistentries.cpp

@@ -60,7 +60,7 @@ DatabaseCommand_LoadPlaylistEntries::exec( DatabaseImpl* dbi )
             m.insert( "resulthint", query.value( 8 ).toString() );
             m.insert( "qid", uuid() );
 
-            Tomahawk::query_ptr q( new Tomahawk::Query( m ) );
+            Tomahawk::query_ptr q = Tomahawk::Query::get( m, false );
             e->setQuery( q );
 
             entrymap.insert( e->guid(), e );

src/libtomahawk/database/databasecommand_logplayback.cpp

@@ -5,7 +5,6 @@
 #include "collection.h"
 #include "database/database.h"
 #include "databaseimpl.h"
-#include "pipeline.h"
 #include "network/servent.h"
 
 using namespace Tomahawk;
@@ -26,9 +25,7 @@ DatabaseCommand_LogPlayback::postCommitHook()
     m.insert( "track", m_track );
     m.insert( "artist", m_artist );
     m.insert( "qid", uuid() );
-    Tomahawk::query_ptr q( new Tomahawk::Query( m ) );
+    Tomahawk::query_ptr q = Tomahawk::Query::get( m );
-
-    Tomahawk::Pipeline::instance()->add( q );
 
     if ( m_action == Finished )
     {

src/libtomahawk/database/databasecommand_playbackhistory.cpp

@@ -3,7 +3,7 @@
 #include <QSqlQuery>
 
 #include "databaseimpl.h"
-#include "pipeline.h"
+
 
 void
 DatabaseCommand_PlaybackHistory::exec( DatabaseImpl* dbi )
@@ -49,7 +49,7 @@ DatabaseCommand_PlaybackHistory::exec( DatabaseImpl* dbi )
             m.insert( "artist", query_track.value( 1 ).toString() );
             m.insert( "qid", uuid() );
 
-            Tomahawk::query_ptr q( new Tomahawk::Query( m ) );
+            Tomahawk::query_ptr q = Tomahawk::Query::get( m );
             ql << q;
         }
     }
@@ -57,8 +57,5 @@ DatabaseCommand_PlaybackHistory::exec( DatabaseImpl* dbi )
     qDebug() << Q_FUNC_INFO << ql.length();
 
     if ( ql.count() )
-    {
-        Tomahawk::Pipeline::instance()->add( ql );
         emit tracks( ql );
-    }
 }

src/libtomahawk/database/databasecommand_resolve.cpp

@@ -8,10 +8,9 @@
 using namespace Tomahawk;
 
 
-DatabaseCommand_Resolve::DatabaseCommand_Resolve( const QVariant& v, bool searchlocal )
+DatabaseCommand_Resolve::DatabaseCommand_Resolve( const QVariant& v )
     : DatabaseCommand()
     , m_v( v )
-    , m_searchlocal( searchlocal )
 {
 }
 
@@ -72,6 +71,7 @@ DatabaseCommand_Resolve::exec( DatabaseImpl* lib )
     if( artists.length() == 0 || tracks.length() == 0 )
     {
         //qDebug() << "No candidates found in first pass, aborting resolve" << artistname << trackname;
+        emit results( qid, res );
         return;
     }
 
@@ -98,11 +98,9 @@ DatabaseCommand_Resolve::exec( DatabaseImpl* lib )
                             "WHERE "
                             "artist.id = file_join.artist AND "
                             "track.id = file_join.track AND "
-                            "file.source %1 AND "
                             "file.id = file_join.file AND "
-                            "file_join.artist IN (%2) AND "
+                            "file_join.artist IN (%1) AND "
-                            "file_join.track IN (%3)"
+                            "file_join.track IN (%2)" )
-        ).arg( m_searchlocal ? "IS NULL" : " > 0 " )
          .arg( artsl.join( "," ) )
          .arg( trksl.join( "," ) );
 
@@ -130,7 +128,7 @@ DatabaseCommand_Resolve::exec( DatabaseImpl* lib )
 
         source_ptr s;
         const QString url_str = files_query.value( 0 ).toString();
-        if( m_searchlocal )
+        if( files_query.value( 13 ).toUInt() == 0 )
         {
             s = SourceList::instance()->getLocal();
             m["url"] = url_str;

src/libtomahawk/database/databasecommand_resolve.h

@@ -13,7 +13,7 @@ class DLLEXPORT DatabaseCommand_Resolve : public DatabaseCommand
 {
 Q_OBJECT
 public:
-    explicit DatabaseCommand_Resolve( const QVariant& v, bool searchlocal );
+    explicit DatabaseCommand_Resolve( const QVariant& v );
 
     virtual QString commandname() const { return "dbresolve"; }
     virtual bool doesMutates() const { return false; }
@@ -27,7 +27,6 @@ public slots:
 
 private:
     QVariant m_v;
-    bool m_searchlocal;
 
     float how_similar( const QVariantMap& q, const QVariantMap& r );
     static int levenshtein( const QString& source, const QString& target );

src/libtomahawk/database/databasecommand_updatesearchindex.cpp

@@ -1,117 +1,39 @@
 #include "databasecommand_updatesearchindex.h"
 
 
-DatabaseCommand_UpdateSearchIndex::DatabaseCommand_UpdateSearchIndex( const QString& t, int p )
+DatabaseCommand_UpdateSearchIndex::DatabaseCommand_UpdateSearchIndex()
     : DatabaseCommand()
-    , table( t )
-    , pkey( p )
 {
-    if( table != "artist" && table != "track" && table != "album" )
-    {
-        Q_ASSERT(false);
-        return;
-    }
 }
 
 
-void DatabaseCommand_UpdateSearchIndex::exec(DatabaseImpl *db)
+void
+DatabaseCommand_UpdateSearchIndex::indexTable( DatabaseImpl* db, const QString& table )
 {
     qDebug() << Q_FUNC_INFO;
-
+    
-    if( table != "artist" && table != "track" && table != "album" )
-    {
-        Q_ASSERT(false);
-        return;
-    }
-
-    // if pkey is 0, consult DB to see what needs indexing
-    if( pkey == 0 )
-    {
-        TomahawkSqlQuery q = db->newquery();
-        q.exec( QString("SELECT coalesce(max(id),0) from %1_search_index").arg(table) );
-        q.next();
-        pkey = 1 + q.value(0).toInt();
-        qDebug() << "updateSearchIndex" << table << "consulted DB, starting at" << pkey;
-    }
-
     TomahawkSqlQuery query = db->newquery();
-    qDebug() << "Building index for" << table << ">= id" << pkey;
+    qDebug() << "Building index for" << table;
-    QString searchtable( table + "_search_index" );
+    query.exec( QString( "SELECT id, name FROM %1" ).arg( table ) );
-    query.exec(QString( "SELECT id, sortname FROM %1 WHERE id >= %2" ).arg( table ).arg(pkey ) );
+    
-
+    QMap< unsigned int, QString > fields;
-    TomahawkSqlQuery upq = db->newquery();
+    while ( query.next() )
-    TomahawkSqlQuery inq = db->newquery();
-    inq.prepare( "INSERT INTO "+ searchtable +" (ngram, id, num) VALUES (?,?,?)" );
-    upq.prepare( "UPDATE "+ searchtable +" SET num=num+? WHERE ngram=? AND id=?" );
-
-    int num_names = 0;
-    int num_ngrams = 0;
-    int id;
-    QString name;
-    QMap<QString, int> ngrammap;
-
-    // this is the new ngram map we build up, to be merged into the
-    // main one in FuzzyIndex:
-    QHash< QString, QMap<quint32, quint16> > idx;
-
-    while( query.next() )
     {
-        id = query.value( 0 ).toInt();
+        fields.insert( query.value( 0 ).toUInt(), query.value( 1 ).toString() );
-        name = query.value( 1 ).toString();
-        num_names++;
-        inq.bindValue( 1, id ); // set id
-        upq.bindValue( 2, id ); // set id
-        ngrammap = DatabaseImpl::ngrams( name );
-        QMapIterator<QString, int> i( ngrammap );
-
-        while ( i.hasNext() )
-        {
-            i.next();
-            num_ngrams++;
-            upq.bindValue( 0, i.value() ); //num
-            upq.bindValue( 1, i.key() ); // ngram
-            upq.exec();
-
-            if( upq.numRowsAffected() == 0 )
-            {
-                inq.bindValue( 0, i.key() ); //ngram
-                inq.bindValue( 2, i.value() ); //num
-                inq.exec();
-                if( inq.numRowsAffected() == 0 )
-                {
-                    qDebug() << "Error updating search index:" << id << name;
-                    continue;
-                }
-            }
-
-            // update ngram cache:
-            QMapIterator<QString, int> iter( ngrammap );
-            while ( iter.hasNext() )
-            {
-                iter.next();
-                if( idx.contains( iter.key() ) )
-                {
-                    idx[ iter.key() ][ id ] += iter.value();
-                }
-                else
-                {
-                    QMap<quint32, quint16> tmp;
-                    tmp.insert( id, iter.value() );
-                    idx.insert( iter.key(), tmp );
-                }
-            }
-
-
-        }
     }
-
+    
-    // merge in our ngrams into the main index
+    db->m_fuzzyIndex->appendFields( table, fields );
-    QMetaObject::invokeMethod( &(db->m_fuzzyIndex),
+}
-                               "mergeIndex",
+
-                               Qt::QueuedConnection,
+
-                               Q_ARG( QString, table ),
+void
-                               QGenericArgument( "QHash< QString, QMap<quint32, quint16> >", &idx )
+DatabaseCommand_UpdateSearchIndex::exec( DatabaseImpl* db )
-                              );
+{
-
+    db->m_fuzzyIndex->beginIndexing();
-    qDebug() << "Finished indexing" << num_names <<" names," << num_ngrams << "ngrams.";
+    
+    indexTable( db, "artist" );
+    indexTable( db, "album" );
+    indexTable( db, "track" );
+
+    db->m_fuzzyIndex->endIndexing();
 }

src/libtomahawk/database/databasecommand_updatesearchindex.h

@@ -10,21 +10,19 @@ class DLLEXPORT DatabaseCommand_UpdateSearchIndex : public DatabaseCommand
 {
 Q_OBJECT
 public:
-    explicit DatabaseCommand_UpdateSearchIndex(const QString& table, int pkey);
+    explicit DatabaseCommand_UpdateSearchIndex();
 
     virtual QString commandname() const { return "updatesearchindex"; }
     virtual bool doesMutates() const { return true; }
-    virtual void exec(DatabaseImpl* db);
+    virtual void exec( DatabaseImpl* db );
 
 signals:
     void indexUpdated();
 
-public slots:
-
 private:
+    void indexTable( DatabaseImpl* db, const QString& table );
+    
     QString table;
-    int pkey;
-
 };
 
 #endif // DATABASECOMMAND_UPDATESEARCHINDEX_H

src/libtomahawk/database/databaseimpl.cpp

@@ -16,7 +16,7 @@
 */
 #include "schema.sql.h"
 
-#define CURRENT_SCHEMA_VERSION 16
+#define CURRENT_SCHEMA_VERSION 18
 
 
 DatabaseImpl::DatabaseImpl( const QString& dbname, Database* parent )
@@ -24,9 +24,8 @@ DatabaseImpl::DatabaseImpl( const QString& dbname, Database* parent )
     , m_lastartid( 0 )
     , m_lastalbid( 0 )
     , m_lasttrkid( 0 )
-    , m_fuzzyIndex( *this )
 {
-    connect( this, SIGNAL(indexReady()), parent, SIGNAL(indexReady()) );
+    connect( this, SIGNAL( indexReady() ), parent, SIGNAL( indexReady() ) );
 
     db = QSqlDatabase::addDatabase( "QSQLITE", "tomahawk" );
     db.setDatabaseName( dbname );
@@ -96,6 +95,8 @@ DatabaseImpl::DatabaseImpl( const QString& dbname, Database* parent )
 
     // in case of unclean shutdown last time:
     query.exec( "UPDATE source SET isonline = 'false'" );
+
+    m_fuzzyIndex = new FuzzyIndex( *this );
 }
 
 
@@ -103,6 +104,8 @@ DatabaseImpl::~DatabaseImpl()
 {
     m_indexThread.quit();
     m_indexThread.wait( 5000 );
+
+    delete m_fuzzyIndex;
 }
 
 
@@ -110,17 +113,17 @@ void
 DatabaseImpl::loadIndex()
 {
     // load ngram index in the background
-    m_fuzzyIndex.moveToThread( &m_indexThread );
+    m_fuzzyIndex->moveToThread( &m_indexThread );
-    connect( &m_indexThread, SIGNAL(started()), &m_fuzzyIndex, SLOT(loadNgramIndex()) );
+    connect( &m_indexThread, SIGNAL( started() ), m_fuzzyIndex, SLOT( loadLuceneIndex() ) );
-    connect( &m_fuzzyIndex, SIGNAL(indexReady()), this, SIGNAL(indexReady()) );
+    connect( m_fuzzyIndex, SIGNAL( indexReady() ), this, SIGNAL( indexReady() ) );
     m_indexThread.start();
 }
 
 
 void
-DatabaseImpl::updateSearchIndex( const QString& table, int pkey )
+DatabaseImpl::updateSearchIndex()
 {
-    DatabaseCommand* cmd = new DatabaseCommand_UpdateSearchIndex(table, pkey);
+    DatabaseCommand* cmd = new DatabaseCommand_UpdateSearchIndex();
     Database::instance()->enqueue( QSharedPointer<DatabaseCommand>( cmd ) );
 }
 
@@ -324,41 +327,24 @@ DatabaseImpl::albumId( int artistid, const QString& name_orig, bool& isnew )
 
 
 QList< int >
-DatabaseImpl::searchTable( const QString& table, const QString& name_orig, uint limit )
+DatabaseImpl::searchTable( const QString& table, const QString& name, uint limit )
 {
     QList< int > results;
     if( table != "artist" && table != "track" && table != "album" )
         return results;
 
-    QString name = sortname( name_orig );
+    QMap< int, float > resultsmap = m_fuzzyIndex->search( table, name );
 
-    // first check for exact matches:
+    QList< QPair<int, float> > resultslist;
-    TomahawkSqlQuery query = newquery();
+    foreach( int i, resultsmap.keys() )
-    query.prepare( QString( "SELECT id FROM %1 WHERE sortname = ?" ).arg( table ) );
-    query.addBindValue( name );
-    query.exec();
-
-    while( query.next() )
-        results.append( query.value( 0 ).toInt() );
-
-    // ngram stuff only works on tracks over a certain length atm:
-    if( name_orig.length() > 3 )
     {
-        // consult ngram index to find candidates:        
+        resultslist << QPair<int, float>( i, (float)resultsmap.value( i ) );
-        QMap< int, float > resultsmap = m_fuzzyIndex.search( table, name );
+    }
+    qSort( resultslist.begin(), resultslist.end(), DatabaseImpl::scorepairSorter );
 
-        //qDebug() << "results map for" << table << resultsmap.size();
+    for( int k = 0; k < resultslist.count(); k++ )
-        QList< QPair<int,float> > resultslist;
+    {
-        foreach( int i, resultsmap.keys() )
+        results << resultslist.at( k ).first;
-        {
-            resultslist << QPair<int,float>( i, (float)resultsmap.value( i ) );
-        }
-        qSort( resultslist.begin(), resultslist.end(), DatabaseImpl::scorepairSorter );
-
-        for( int k = 0; k < resultslist.size() && k < (int)limit; ++k )
-        {
-            results << resultslist.at( k ).first;
-        }
     }
 
     return results;
@@ -383,35 +369,6 @@ DatabaseImpl::getTrackFids( int tid )
 }
 
 
-QMap< QString, int >
-DatabaseImpl::ngrams( const QString& str_orig )
-{
-    static QMap< QString, QMap<QString, int> > memo;
-    if( memo.contains( str_orig ) )
-        return memo.value( str_orig );
-
-    int n = 3;
-    QMap<QString, int> ret;
-    QString str( " " + DatabaseImpl::sortname( str_orig ) + " " );
-    int num = str.length();
-    QString ngram;
-
-    for( int j = 0; j < num - ( n - 1 ); j++ )
-    {
-         ngram = str.mid( j, n );
-         Q_ASSERT( ngram.length() == n );
-
-         if( ret.contains( ngram ) )
-             ret[ngram]++;
-         else
-             ret[ngram] = 1;
-    }
-
-    memo.insert( str_orig, ret );
-    return ret;
-}
-
-
 QString
 DatabaseImpl::sortname( const QString& str )
 {

src/libtomahawk/database/databaseimpl.h

@@ -36,10 +36,9 @@ public:
     int trackId( int artistid, const QString& name_orig, bool& isnew );
     int albumId( int artistid, const QString& name_orig, bool& isnew );
 
-    QList< int > searchTable( const QString& table, const QString& name_orig, uint limit = 10 );
+    QList< int > searchTable( const QString& table, const QString& name, uint limit = 10 );
     QList< int > getTrackFids( int tid );
 
-    static QMap<QString,int> ngrams( const QString& str_orig );
     static QString sortname( const QString& str );
 
     QVariantMap artist( int id );
@@ -54,7 +53,7 @@ public:
     }
 
     // indexes entries from "table" where id >= pkey
-    void updateSearchIndex( const QString& table, int pkey );
+    void updateSearchIndex();
 
     const QString& dbid() const { return m_dbid; }
 
@@ -76,7 +75,7 @@ private:
     QString m_dbid;
 
     QThread m_indexThread;
-    FuzzyIndex m_fuzzyIndex;
+    FuzzyIndex* m_fuzzyIndex;
 };
 
 #endif // DATABASEIMPL_H

src/libtomahawk/database/databaseresolver.cpp

@@ -4,9 +4,8 @@
 #include "database/database.h"
 #include "database/databasecommand_resolve.h"
 
-DatabaseResolver::DatabaseResolver( bool searchlocal, int weight )
+DatabaseResolver::DatabaseResolver( int weight )
     : Resolver()
-    , m_searchlocal( searchlocal )
     , m_weight( weight )
 {
 }
@@ -17,13 +16,7 @@ DatabaseResolver::resolve( const QVariant& v )
 {
     //qDebug() << Q_FUNC_INFO << v;
 
-    if( !m_searchlocal )
+    DatabaseCommand_Resolve* cmd = new DatabaseCommand_Resolve( v );
-    {
-        if( Servent::instance()->numConnectedPeers() == 0 )
-            return;
-    }
-
-    DatabaseCommand_Resolve* cmd = new DatabaseCommand_Resolve( v, m_searchlocal );
 
     connect( cmd, SIGNAL( results( Tomahawk::QID, QList< Tomahawk::result_ptr> ) ),
                     SLOT( gotResults( Tomahawk::QID, QList< Tomahawk::result_ptr> ) ), Qt::QueuedConnection );
@@ -45,5 +38,5 @@ DatabaseResolver::gotResults( const Tomahawk::QID qid, QList< Tomahawk::result_p
 QString
 DatabaseResolver::name() const
 {
-    return QString( "Database (%1)" ).arg( m_searchlocal ? "local" : "remote" );
+    return QString( "DatabaseResolver" );
 }

src/libtomahawk/database/databaseresolver.h

@@ -13,7 +13,7 @@ class DLLEXPORT DatabaseResolver : public Tomahawk::Resolver
 Q_OBJECT
 
 public:
-    explicit DatabaseResolver( bool searchlocal, int weight );
+    explicit DatabaseResolver( int weight );
 
     virtual QString name() const;
     virtual unsigned int weight() const { return m_weight; }
@@ -26,7 +26,6 @@ private slots:
     void gotResults( const Tomahawk::QID qid, QList< Tomahawk::result_ptr> results );
 
 private:
-    bool m_searchlocal;
     int m_weight;
 };
 

src/libtomahawk/database/databaseworker.cpp

@@ -40,9 +40,10 @@ DatabaseWorker::run()
 void
 DatabaseWorker::enqueue( const QSharedPointer<DatabaseCommand>& cmd )
 {
+    m_outstanding++;
+
     QMutexLocker lock( &m_mut );
     m_commands << cmd;
-    m_outstanding++;
 
     if ( m_outstanding == 1 )
         QTimer::singleShot( 0, this, SLOT( doWork() ) );
@@ -163,11 +164,7 @@ DatabaseWorker::doWork()
 
     cmd->emitFinished();
 
-    {
+    m_outstanding--;
-        QMutexLocker lock( &m_mut );
-        m_outstanding--;
-    }
-
     if ( m_outstanding > 0 )
         QTimer::singleShot( 0, this, SLOT( doWork() ) );
 }

src/libtomahawk/database/fuzzyindex.cpp

@@ -1,139 +1,146 @@
 #include "fuzzyindex.h"
 
 #include "databaseimpl.h"
+#include "utils/tomahawkutils.h"
 
+#include <QDir>
 #include <QTime>
 
+#include <CLucene.h>
 
-FuzzyIndex::FuzzyIndex( DatabaseImpl &db )
+
+FuzzyIndex::FuzzyIndex( DatabaseImpl& db )
     : QObject()
     , m_db( db )
-    , m_loaded( false )
+    , m_luceneReader( 0 )
+    , m_luceneSearcher( 0 )
 {
+    QString lucenePath = TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" );
+    bool create = !lucene::index::IndexReader::indexExists( lucenePath.toStdString().c_str() );
+    m_luceneDir = lucene::store::FSDirectory::getDirectory( lucenePath.toStdString().c_str(), create );
+
+    m_analyzer = new lucene::analysis::SimpleAnalyzer();
+}
+
+
+FuzzyIndex::~FuzzyIndex()
+{
+    delete m_luceneSearcher;
+    delete m_luceneReader;
+    delete m_analyzer;
+    delete m_luceneDir;
 }
 
 
 void
-FuzzyIndex::loadNgramIndex()
+FuzzyIndex::beginIndexing()
 {
-    // this updates the index in the DB, if needed:
+    lucene::index::IndexWriter luceneWriter = lucene::index::IndexWriter( m_luceneDir, m_analyzer, true );
-    qDebug() << "Checking catalogue is fully indexed...";
+    m_mutex.lock();
-    m_db.updateSearchIndex( "artist", 0 );
+}
-    m_db.updateSearchIndex( "album", 0 );
-    m_db.updateSearchIndex( "track", 0 );
 
-    // loads index from DB into memory:
+
-    qDebug() << "Loading search index for catalogue metadata..." << thread();
+void
-    loadNgramIndex_helper( m_artist_ngrams, "artist" );
+FuzzyIndex::endIndexing()
-    loadNgramIndex_helper( m_album_ngrams, "album" );
+{
-    loadNgramIndex_helper( m_track_ngrams, "track" );
+    m_mutex.unlock();
-    m_loaded = true;
+}
+
+
+void
+FuzzyIndex::appendFields( const QString& table, const QMap< unsigned int, QString >& fields )
+{
+    delete m_luceneSearcher;
+    delete m_luceneReader;
+    m_luceneSearcher = 0;
+    m_luceneReader = 0;
+    
+    bool create = !lucene::index::IndexReader::indexExists( TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" ).toStdString().c_str() );
+    lucene::index::IndexWriter luceneWriter = lucene::index::IndexWriter( m_luceneDir, m_analyzer, create );
+    lucene::document::Document doc;
+    
+    QMapIterator< unsigned int, QString > it( fields );
+    while ( it.hasNext() )
+    {
+        it.next();
+        unsigned int id = it.key();
+        QString name = it.value();
+
+        {
+            lucene::document::Field* field = new lucene::document::Field( table.toStdWString().c_str(), name.toStdWString().c_str(),
+                                                                          lucene::document::Field::STORE_YES | lucene::document::Field::INDEX_UNTOKENIZED );
+            doc.add( *field );
+        }
+        
+        {
+            lucene::document::Field* field = new lucene::document::Field( _T( "id" ), QString::number( id ).toStdWString().c_str(),
+            lucene::document::Field::STORE_YES | lucene::document::Field::INDEX_NO );
+            doc.add( *field );
+        }
+        
+        luceneWriter.addDocument( &doc );
+        doc.clear();
+    }
+    
+    luceneWriter.close();
+}
+
+
+void
+FuzzyIndex::loadLuceneIndex()
+{
     emit indexReady();
 }
 
 
-void
-FuzzyIndex::loadNgramIndex_helper( QHash< QString, QMap<quint32, quint16> >& idx, const QString& table, unsigned int fromkey )
-{
-    QTime t;
-    t.start();
-
-    TomahawkSqlQuery query = m_db.newquery();
-    query.exec( QString( "SELECT ngram, id, num "
-                         "FROM %1_search_index "
-                         "WHERE id >= %2 "
-                         "ORDER BY ngram" ).arg( table ).arg( fromkey ) );
-
-    QMap<quint32, quint16> ngram_idx;
-    QString lastngram;
-    while( query.next() )
-    {
-        const QString ng = query.value( 0 ).toString();
-        if( lastngram.isEmpty() )
-            lastngram = ng;
-
-        if( ng != lastngram )
-        {
-            idx.insert( lastngram, ngram_idx );
-            lastngram = ng;
-            ngram_idx.clear();
-        }
-
-        ngram_idx.insert( query.value( 1 ).toUInt(),
-                          query.value( 2 ).toUInt() );
-    }
-
-    idx.insert( lastngram, ngram_idx );
-    qDebug() << "Loaded" << idx.size()
-             << "ngram entries for" << table
-             << "in" << t.elapsed() << "ms";
-}
-
-
-void FuzzyIndex::mergeIndex( const QString& table, const QHash< QString, QMap<quint32, quint16> >& tomerge )
-{
-    qDebug() << Q_FUNC_INFO << table << tomerge.keys().size();
-
-    QHash< QString, QMap<quint32, quint16> >* idx;
-    if     ( table == "artist" ) idx = &m_artist_ngrams;
-    else if( table == "album"  ) idx = &m_album_ngrams;
-    else if( table == "track"  ) idx = &m_track_ngrams;
-    else Q_ASSERT( false );
-
-    if( tomerge.size() == 0 )
-        return;
-
-    if( idx->size() == 0 )
-    {
-        *idx = tomerge;
-    }
-    else
-    {
-        QList<QString> tmk = tomerge.keys();
-        foreach( const QString& ngram, tmk )
-        {
-            if( idx->contains( ngram ) )
-            {
-                QList<unsigned int> tmkn = tomerge[ngram].keys();
-                foreach( quint32 id, tmkn )
-                {
-                    (*idx)[ ngram ][ id ] += tomerge[ngram][id];
-                }
-            }
-            else
-            {
-                idx->insert( ngram, tomerge[ngram] );
-            }
-        }
-    }
-
-    qDebug() << Q_FUNC_INFO << table << "merge complete, num items:" << tomerge.size();
-}
-
-
 QMap< int, float >
 FuzzyIndex::search( const QString& table, const QString& name )
 {
+    QMutexLocker lock( &m_mutex );
+
     QMap< int, float > resultsmap;
-
+    if ( !m_luceneReader )
-    QHash< QString, QMap<quint32, quint16> >* idx;
-    if( table == "artist" ) idx = &m_artist_ngrams;
-    else if( table == "album" ) idx = &m_album_ngrams;
-    else if( table == "track" ) idx = &m_track_ngrams;
-
-    QMap<QString,int> ngramsmap = DatabaseImpl::ngrams( name );
-    foreach( const QString& ngram, ngramsmap.keys() )
     {
-        if( !idx->contains( ngram ) )
+        if ( !lucene::index::IndexReader::indexExists( TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" ).toStdString().c_str() ) )
-            continue;
-
-        //qDebug() << name_orig << "NGRAM:" << ngram << "candidates:" << (*idx)[ngram].size();
-        QMapIterator<quint32, quint16> iter( (*idx)[ngram] );
-        while( iter.hasNext() )
         {
-            iter.next();
+            qDebug() << Q_FUNC_INFO << "index didn't exist.";
-            resultsmap[ (int) iter.key() ] += (float) iter.value();
+            return resultsmap;
+        }
+
+        m_luceneReader = lucene::index::IndexReader::open( m_luceneDir );
+        m_luceneSearcher = new lucene::search::IndexSearcher( m_luceneReader );
+    }
+
+    if ( name.isEmpty() )
+        return resultsmap;
+
+    lucene::analysis::SimpleAnalyzer analyzer;
+    lucene::queryParser::QueryParser parser( table.toStdWString().c_str(), m_analyzer );
+    lucene::search::Hits* hits = 0;
+
+    lucene::search::FuzzyQuery* qry = new lucene::search::FuzzyQuery( new lucene::index::Term( table.toStdWString().c_str(),
+                                                                                               name.toStdWString().c_str() ) );
+    hits = m_luceneSearcher->search( qry );
+
+    for ( unsigned int i = 0; i < hits->length(); i++ )
+    {
+        lucene::document::Document* d = &hits->doc( i );
+
+        float score = hits->score( i );
+        int id = QString::fromWCharArray( d->get( _T( "id" ) ) ).toInt();
+        QString result = QString::fromWCharArray( d->get( table.toStdWString().c_str() ) );
+
+        if ( result.toLower() == name.toLower() )
+            score = 1.0;
+        else
+            score = qMin( score, (float)0.99 );
+
+        if ( score > 0.05 )
+        {
+            resultsmap.insert( id, score );
+//            qDebug() << "Hitres:" << result << id << score << table << name;
         }
     }
+
     return resultsmap;
 }

src/libtomahawk/database/fuzzyindex.h

@@ -5,33 +5,59 @@
 #include <QMap>
 #include <QHash>
 #include <QString>
+#include <QMutex>
+
+namespace lucene
+{
+    namespace analysis
+    {
+        class SimpleAnalyzer;
+    }
+    namespace store
+    {
+        class Directory;
+    }
+    namespace index
+    {
+        class IndexReader;
+        class IndexWriter;
+    }
+    namespace search
+    {
+        class IndexSearcher;
+    }
+}
 
 class DatabaseImpl;
 
-
 class FuzzyIndex : public QObject
 {
 Q_OBJECT
 
 public:
-    explicit FuzzyIndex( DatabaseImpl &db );
+    explicit FuzzyIndex( DatabaseImpl& db );
+    ~FuzzyIndex();
 
+    void beginIndexing();
+    void endIndexing();
+    void appendFields( const QString& table, const QMap< unsigned int, QString >& fields );
+    
 signals:
     void indexReady();
 
 public slots:
-    void loadNgramIndex();
+    void loadLuceneIndex();
+
     QMap< int, float > search( const QString& table, const QString& name );
-    void mergeIndex( const QString& table, const QHash< QString, QMap<quint32, quint16> >& tomerge );
 
 private:
-    void loadNgramIndex_helper( QHash< QString, QMap<quint32, quint16> >& idx, const QString& table, unsigned int fromkey = 0 );
+    DatabaseImpl& m_db;
+    QMutex m_mutex;
 
-    // maps an ngram to {track id, num occurences}
+    lucene::analysis::SimpleAnalyzer* m_analyzer;
-    QHash< QString, QMap<quint32, quint16> > m_artist_ngrams, m_album_ngrams, m_track_ngrams;
+    lucene::store::Directory* m_luceneDir;
-
+    lucene::index::IndexReader* m_luceneReader;
-    DatabaseImpl & m_db;
+    lucene::search::IndexSearcher* m_luceneSearcher;
-    bool m_loaded;
 };
 
 #endif // FUZZYINDEX_H

src/libtomahawk/database/gen_schema.h.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+
+# !!!! You need to manually generate schema.sql.h when the schema changes:
+# ./gen_schema.h.sh ./schema.sql tomahawk > schema.sql.h
+
 schema=$1
 name=$2
 

src/libtomahawk/database/schema.sql

@@ -14,6 +14,8 @@ CREATE TABLE IF NOT EXISTS oplog (
 CREATE UNIQUE INDEX oplog_guid ON oplog(guid);
 CREATE INDEX oplog_source ON oplog(source);
 
+
+
 -- the basic 3 catalogue tables:
 
 CREATE TABLE IF NOT EXISTS artist (
@@ -39,6 +41,8 @@ CREATE TABLE IF NOT EXISTS album (
 );
 CREATE UNIQUE INDEX album_artist_sortname ON album(artist,sortname);
 
+
+
 -- Source, typically a remote peer.
 
 CREATE TABLE IF NOT EXISTS source (
@@ -51,6 +55,7 @@ CREATE TABLE IF NOT EXISTS source (
 CREATE UNIQUE INDEX source_name ON source(name);
 
 
+
 -- playlists
 
 CREATE TABLE IF NOT EXISTS playlist (
@@ -64,9 +69,6 @@ CREATE TABLE IF NOT EXISTS playlist (
     currentrevision TEXT REFERENCES playlist_revision(guid) DEFERRABLE INITIALLY DEFERRED
 );
 
---INSERT INTO playlist(guid, title, info, currentrevision) 
--- VALUES('playlistguid-1','Test Playlist','this playlist automatically created and used for testing','revisionguid-1');
-
 CREATE TABLE IF NOT EXISTS playlist_item (
     guid TEXT PRIMARY KEY,
     playlist TEXT NOT NULL REFERENCES playlist(guid) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
@@ -79,16 +81,8 @@ CREATE TABLE IF NOT EXISTS playlist_item (
     addedby INTEGER REFERENCES source(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED, -- who added this to the playlist
     result_hint TEXT        -- hint as to a result, to avoid using the resolver
 );
-
 CREATE INDEX playlist_item_playlist ON playlist_item(playlist);
 
---INSERT INTO playlist_item(guid, playlist, trackname, artistname) 
---       VALUES('itemguid-1','playlistguid-1','track name 01','artist name 01');
---INSERT INTO playlist_item(guid, playlist, trackname, artistname) 
---       VALUES('itemguid-2','playlistguid-1','track name 02','artist name 02');
---INSERT INTO playlist_item(guid, playlist, trackname, artistname) 
---       VALUES('itemguid-3','playlistguid-1','track name 03','artist name 03');
---
 CREATE TABLE IF NOT EXISTS playlist_revision (
     guid TEXT PRIMARY KEY,
     playlist TEXT NOT NULL REFERENCES playlist(guid) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
@@ -99,31 +93,6 @@ CREATE TABLE IF NOT EXISTS playlist_revision (
 );
 
 
--- the trigram search indexes
-
-CREATE TABLE IF NOT EXISTS artist_search_index (
-    ngram TEXT NOT NULL,
-    id INTEGER NOT NULL REFERENCES artist(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
-    num INTEGER NOT NULL DEFAULT 1,
-    PRIMARY KEY(ngram, id)
-);
--- CREATE INDEX artist_search_index_ngram ON artist_search_index(ngram);
-
-CREATE TABLE IF NOT EXISTS album_search_index (
-    ngram TEXT NOT NULL,
-    id INTEGER NOT NULL REFERENCES album(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
-    num INTEGER NOT NULL DEFAULT 1,
-    PRIMARY KEY(ngram, id)
-);
--- CREATE INDEX album_search_index_ngram ON album_search_index(ngram);
-
-CREATE TABLE IF NOT EXISTS track_search_index (
-    ngram TEXT NOT NULL,
-    id INTEGER NOT NULL REFERENCES track(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
-    num INTEGER NOT NULL DEFAULT 1,
-    PRIMARY KEY(ngram, id)
-); 
--- CREATE INDEX track_search_index_ngram ON track_search_index(ngram);
 
 -- files on disk and joinage with catalogue. physical properties of files only:
 
@@ -149,7 +118,6 @@ CREATE TABLE IF NOT EXISTS dirs_scanned (
     mtime INTEGER NOT NULL
 );
 
-
 CREATE TABLE IF NOT EXISTS file_join (
     file INTEGER PRIMARY KEY REFERENCES file(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
     artist INTEGER NOT NULL REFERENCES artist(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
@@ -164,9 +132,9 @@ CREATE INDEX file_join_album  ON file_join(album);
 
 
 -- tags, weighted and by source (rock, jazz etc)
+
 -- weight is always 1.0 if tag provided by our user.
 -- may be less from aggregate sources like lastfm global tags
-
 CREATE TABLE IF NOT EXISTS track_tags (
     id INTEGER PRIMARY KEY,   -- track id
     source INTEGER REFERENCES source(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
@@ -209,6 +177,7 @@ CREATE INDEX track_attrib_id ON track_attributes(id);
 CREATE INDEX track_attrib_k  ON track_attributes(k);
 
 
+
 -- playback history
 
 -- if source=null, file is local to this machine
@@ -223,10 +192,11 @@ CREATE INDEX playback_log_source ON playback_log(source);
 CREATE INDEX playback_log_track ON playback_log(track);
 
 
+
 -- Schema version, and misc tomahawk settings relating to the collection db
 
 CREATE TABLE IF NOT EXISTS settings (
     k TEXT NOT NULL PRIMARY KEY,
     v TEXT NOT NULL DEFAULT ''
 );
-INSERT INTO settings(k,v) VALUES('schema_version', '16');
+INSERT INTO settings(k,v) VALUES('schema_version', '18');

src/libtomahawk/database/schema.sql.h

@@ -1,5 +1,5 @@
 /*
-    This file was automatically generated from schema.sql on Thu Jan  6 09:52:57 CET 2011.
+    This file was automatically generated from ./schema.sql on Sat Jan 29 10:28:18 CET 2011.
 */
 
 static const char * tomahawk_schema_sql = 
@@ -73,24 +73,6 @@ static const char * tomahawk_schema_sql =
 "    timestamp INTEGER NOT NULL DEFAULT 0,"
 "    previous_revision TEXT REFERENCES playlist_revision(guid) DEFERRABLE INITIALLY DEFERRED"
 ");"
-"CREATE TABLE IF NOT EXISTS artist_search_index ("
-"    ngram TEXT NOT NULL,"
-"    id INTEGER NOT NULL REFERENCES artist(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,"
-"    num INTEGER NOT NULL DEFAULT 1,"
-"    PRIMARY KEY(ngram, id)"
-");"
-"CREATE TABLE IF NOT EXISTS album_search_index ("
-"    ngram TEXT NOT NULL,"
-"    id INTEGER NOT NULL REFERENCES album(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,"
-"    num INTEGER NOT NULL DEFAULT 1,"
-"    PRIMARY KEY(ngram, id)"
-");"
-"CREATE TABLE IF NOT EXISTS track_search_index ("
-"    ngram TEXT NOT NULL,"
-"    id INTEGER NOT NULL REFERENCES track(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,"
-"    num INTEGER NOT NULL DEFAULT 1,"
-"    PRIMARY KEY(ngram, id)"
-");"
 "CREATE TABLE IF NOT EXISTS file ("
 "    id INTEGER PRIMARY KEY AUTOINCREMENT,"
 "    source INTEGER REFERENCES source(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,"
@@ -162,7 +144,7 @@ static const char * tomahawk_schema_sql =
 "    k TEXT NOT NULL PRIMARY KEY,"
 "    v TEXT NOT NULL DEFAULT ''"
 ");"
-"INSERT INTO settings(k,v) VALUES('schema_version', '16');"
+"INSERT INTO settings(k,v) VALUES('schema_version', '18');"
     ;
 
 const char * get_tomahawk_sql()

src/libtomahawk/pipeline.cpp

@@ -29,7 +29,7 @@ Pipeline::Pipeline( QObject* parent )
 void
 Pipeline::databaseReady()
 {
-    connect( Database::instance(), SIGNAL(indexReady()), this, SLOT(indexReady()), Qt::QueuedConnection );
+    connect( Database::instance(), SIGNAL( indexReady() ), this, SLOT( indexReady() ), Qt::QueuedConnection );
     Database::instance()->loadIndex();
 }
 
@@ -73,7 +73,7 @@ Pipeline::addResolver( Resolver* r, bool sort )
 
 
 void
-Pipeline::add( const QList<query_ptr>& qlist, bool prioritized )
+Pipeline::resolve( const QList<query_ptr>& qlist, bool prioritized )
 {
     {
         QMutexLocker lock( &m_mut );
@@ -103,12 +103,21 @@ Pipeline::add( const QList<query_ptr>& qlist, bool prioritized )
 
 
 void
-Pipeline::add( const query_ptr& q, bool prioritized )
+Pipeline::resolve( const query_ptr& q, bool prioritized )
 {
-    //qDebug() << Q_FUNC_INFO << (qlonglong)q.data() << q->toString();
+    if ( q.isNull() )
+        return;
+    
     QList< query_ptr > qlist;
     qlist << q;
-    add( qlist, prioritized );
+    resolve( qlist, prioritized );
+}
+
+
+void
+Pipeline::resolve( QID qid, bool prioritized )
+{
+    resolve( query( qid ), prioritized );
 }
 
 
@@ -117,12 +126,22 @@ Pipeline::reportResults( QID qid, const QList< result_ptr >& results )
 {
     QMutexLocker lock( &m_mut );
 
-    if( !m_qids.contains( qid ) )
+    if ( !m_qids.contains( qid ) )
     {
         qDebug() << "reportResults called for unknown QID";
         return;
     }
 
+    unsigned int state = m_qidsState.value( qid ) - 1;
+    m_qidsState.insert( qid, state );
+
+    if ( state == 0 )
+    {
+        // All resolvers have reported back their results for this query now
+        const query_ptr& q = m_qids.value( qid );
+        q->onResolvingFinished();
+    }
+    
     if ( !results.isEmpty() )
     {
         //qDebug() << Q_FUNC_INFO << qid;
@@ -189,6 +208,9 @@ Pipeline::shunt( const query_ptr& q )
 
             // resolvers aren't allowed to block in this call:
             //qDebug() << "Dispaching to resolver" << r->name();
+
+            unsigned int state = m_qidsState.value( q->id() );
+            m_qidsState.insert( q->id(), state + 1 );
             r->resolve( q->toVariant() );
         }
         else

src/libtomahawk/pipeline.h

@@ -27,9 +27,6 @@ public:
 
     explicit Pipeline( QObject* parent = 0 );
 
-//    const query_ptr& query( QID qid ) const;
-//    result_ptr result( RID rid ) const;
-
     void reportResults( QID qid, const QList< result_ptr >& results );
 
     /// sorter to rank resolver priority
@@ -49,8 +46,9 @@ public:
     }
 
 public slots:
-    void add( const query_ptr& q, bool prioritized = true );
+    void resolve( const query_ptr& q, bool prioritized = true );
-    void add( const QList<query_ptr>& qlist, bool prioritized = true );
+    void resolve( const QList<query_ptr>& qlist, bool prioritized = true );
+    void resolve( QID qid, bool prioritized = true );
     void databaseReady();
 
 signals:
@@ -64,6 +62,8 @@ private slots:
 
 private:
     QList< Resolver* > m_resolvers;
+
+    QMap< QID, unsigned int > m_qidsState;
     QMap< QID, query_ptr > m_qids;
     QMap< RID, result_ptr > m_rids;
 

src/libtomahawk/playlist.cpp

@@ -20,7 +20,7 @@ using namespace Tomahawk;
 void
 PlaylistEntry::setQueryVariant( const QVariant& v )
 {
-    m_query = query_ptr( new Query( v ) );
+    m_query = Tomahawk::Query::get( v, false );
 }
 
 
@@ -342,7 +342,8 @@ Playlist::resolve()
     {
         qlist << p->query();
     }
-    Pipeline::instance()->add( qlist );
+
+    Pipeline::instance()->resolve( qlist );
 }
 
 

src/libtomahawk/playlist/collectionmodel.cpp

@@ -195,7 +195,7 @@ CollectionModel::onTracksAdded( const QList<QVariant>& tracks, const collection_
     PlItem* plitem;
     foreach( const QVariant& v, tracks )
     {
-        Tomahawk::query_ptr query = query_ptr( new Query( v ) );
+        Tomahawk::query_ptr query = Tomahawk::Query::get( v, false );
 
         // FIXME: needs merging
         // Manually add a result, since it's coming from the local collection

src/libtomahawk/query.cpp

@@ -2,14 +2,28 @@
 
 #include <QtAlgorithms>
 
+#include "database/database.h"
+#include "pipeline.h"
+#include "sourcelist.h"
+
 using namespace Tomahawk;
 
 
+query_ptr
+Query::get( const QVariant& v, bool autoResolve )
+{
+    query_ptr q = query_ptr( new Query( v ) );
+
+    if ( autoResolve )
+        Pipeline::instance()->resolve( q );
+    return q;
+}
+
+
 Query::Query( const QVariant& v )
     : m_v( v )
     , m_solved( false )
 {
-    // ensure a QID is present:
     QVariantMap m = m_v.toMap();
 
     m_artist = m.value( "artist" ).toString();
@@ -17,6 +31,9 @@ Query::Query( const QVariant& v )
     m_track = m.value( "track" ).toString();
 
     m_qid = m.value( "qid" ).toString();
+
+    connect( SourceList::instance(), SIGNAL( sourceAdded( Tomahawk::source_ptr ) ), SLOT( refreshResults() ), Qt::QueuedConnection );
+    connect( Database::instance(), SIGNAL( indexReady() ), SLOT( refreshResults() ), Qt::QueuedConnection );
 }
 
 
@@ -46,6 +63,13 @@ Query::addResults( const QList< Tomahawk::result_ptr >& newresults )
 }
 
 
+void
+Query::refreshResults()
+{
+    Pipeline::instance()->resolve( id() );
+}
+
+
 void
 Query::resultUnavailable()
 {
@@ -83,6 +107,14 @@ Query::removeResult( const Tomahawk::result_ptr& result )
 }
 
 
+void
+Query::onResolvingFinished()
+{
+//    qDebug() << Q_FUNC_INFO << "Finished resolving." << toString();
+    emit resolvingFinished( !m_results.isEmpty() );
+}
+
+
 QList< result_ptr >
 Query::results() const
 {

src/libtomahawk/query.h

@@ -20,6 +20,7 @@ class DLLEXPORT Query : public QObject
 Q_OBJECT
 
 public:
+    static query_ptr get( const QVariant& v, bool autoResolve = true );
     explicit Query( const QVariant& v );
 
     QVariant toVariant() const { return m_v; }
@@ -39,7 +40,7 @@ public:
     bool solved() const { return m_solved; }
 
     unsigned int lastPipelineWeight() const { return m_lastpipelineweight; }
-    void setLastPipelineWeight( unsigned int w ) { m_lastpipelineweight = w;}
+    void setLastPipelineWeight( unsigned int w ) { m_lastpipelineweight = w; }
 
     /// for debug output:
     QString toString() const
@@ -54,15 +55,20 @@ public:
 signals:
     void resultsAdded( const QList<Tomahawk::result_ptr>& );
     void resultsRemoved( const Tomahawk::result_ptr& );
-    void solvedStateChanged( bool state );
 
+    void solvedStateChanged( bool state );
+    void resolvingFinished( bool hasResults );
+    
 public slots:
     /// (indirectly) called by resolver plugins when results are found
     void addResults( const QList< Tomahawk::result_ptr >& );
     void removeResult( const Tomahawk::result_ptr& );
 
+    void onResolvingFinished();
+
 private slots:
     void resultUnavailable();
+    void refreshResults();
 
 private:
     mutable QMutex m_mut;

src/libtomahawk/result.cpp

@@ -59,7 +59,7 @@ Result::toString() const
 Tomahawk::query_ptr
 Result::toQuery() const
 {
-    Tomahawk::query_ptr query = Tomahawk::query_ptr( new Tomahawk::Query( toVariant() ) );
+    Tomahawk::query_ptr query = Tomahawk::Query::get( toVariant(), false );
     return query;
 }
 

src/libtomahawk/utils/xspfloader.cpp

@@ -105,7 +105,7 @@ XSPFLoader::gotBody()
         v.insert( "album", e.firstChildElement( "album" ).text() );
         v.insert( "track", e.firstChildElement( "title" ).text() );
 
-        p->setQuery( Tomahawk::query_ptr(new Tomahawk::Query(v)) );
+        p->setQuery( Tomahawk::Query::get( v ) );
         m_entries << p;
     }
 

src/libtomahawk/widgets/newplaylistwidget.cpp

@@ -11,7 +11,6 @@
 
 #include "widgets/overlaywidget.h"
 
-#include "pipeline.h"
 #include "utils/xspfloader.h"
 
 #include "sourcelist.h"
@@ -115,8 +114,6 @@ NewPlaylistWidget::suggestionsFound()
         ql.append( entry->query() );
     }
 
-    Tomahawk::Pipeline::instance()->add( ql );
-
     loader->deleteLater();
 }
 

src/tomahawkapp.cpp

@@ -306,8 +306,7 @@ void
 TomahawkApp::setupPipeline()
 {
     // setup resolvers for local content, and (cached) remote collection content
-    Pipeline::instance()->addResolver( new DatabaseResolver( true,  100 ) );
+    Pipeline::instance()->addResolver( new DatabaseResolver( 100 ) );
-    Pipeline::instance()->addResolver( new DatabaseResolver( false, 90 ) );
 
 //    new ScriptResolver("/home/rj/src/tomahawk-core/contrib/magnatune/magnatune-resolver.php");
 }

src/tomahawkwindow.cpp

@@ -351,5 +351,5 @@ TomahawkWindow::setWindowTitle( const QString& title )
 void
 TomahawkWindow::showAboutTomahawk()
 {
-    QMessageBox::about( this, "About Tomahawk", "Copyright 2010 Christian Muehlhaeuser <muesli@gmail.com>\n\nThanks to: Leo Franchi, Jeff Mitchell, Dominik Schmidt, Alejandro Wainzinger and Steve Robertson" );
+    QMessageBox::about( this, "About Tomahawk", "Copyright 2010, 2011 Christian Muehlhaeuser <muesli@gmail.com>\n\nThanks to: Leo Franchi, Jeff Mitchell, Dominik Schmidt, Alejandro Wainzinger and Steve Robertson" );
 }

src/web/api_v1.h

@@ -112,8 +112,7 @@ public slots:
         m.insert( "track", event->url.queryItemValue("track") );
         m.insert( "qid", qid );
 
-        Tomahawk::query_ptr qry( new Tomahawk::Query( m ) );
+        Tomahawk::query_ptr qry = Tomahawk::Query::get( m );
-        Tomahawk::Pipeline::instance()->add( qry );
 
         QVariantMap r;
         r.insert( "qid", qid );

src/xmppbot/xmppbot.cpp

@@ -5,7 +5,6 @@
 #include "album.h"
 #include "typedefs.h"
 #include "tomahawksettings.h"
-#include "pipeline.h"
 
 #include "audio/audioengine.h"
 
@@ -145,11 +144,10 @@ void XMPPBot::handleMessage(const Message& msg, MessageSession* session)
         QVariantMap qv;
         qv["artist"] = tokens.first().trimmed();
         qv["track"] = tokens.last().trimmed();
-        Tomahawk::query_ptr q( new Tomahawk::Query( qv ) );
+        Tomahawk::query_ptr q = Tomahawk::Query::get( qv );
         connect( q.data(), SIGNAL( resultsAdded( QList<Tomahawk::result_ptr> ) ),
                              SLOT( onResultsAdded( QList<Tomahawk::result_ptr> ) ) );
 
-        Tomahawk::Pipeline::instance()->add( q );
         return;
     }
     else if ( body.startsWith( "stop" ) )