diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2c0a8fa24..6ca62025a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -16,6 +16,7 @@ ENDIF()
FIND_PACKAGE( Taglib 1.6.0 REQUIRED )
FIND_PACKAGE( LibLastFm 0.3.3 REQUIRED )
FIND_PACKAGE( LibEchonest 1.1.1 REQUIRED )
+FIND_PACKAGE( CLucene REQUIRED )
IF( UNIX AND NOT APPLE )
ADD_SUBDIRECTORY( alsa-playback )
diff --git a/CMakeModules/FindCLucene.cmake b/CMakeModules/FindCLucene.cmake
new file mode 100644
index 000000000..b7c0e862b
--- /dev/null
+++ b/CMakeModules/FindCLucene.cmake
@@ -0,0 +1,97 @@
+#
+# This module looks for clucene (http://clucene.sf.net) support
+# It will define the following values
+#
+# CLUCENE_INCLUDE_DIR = where CLucene/StdHeader.h can be found
+# CLUCENE_LIBRARY_DIR = where CLucene/clucene-config.h can be found
+# CLUCENE_LIBRARY = the library to link against CLucene
+# CLUCENE_VERSION = The CLucene version string
+# CLucene_FOUND = set to 1 if clucene is found
+#
+
+INCLUDE(CheckSymbolExists)
+INCLUDE(FindLibraryWithDebug)
+
+if(NOT CLUCENE_MIN_VERSION)
+ set(CLUCENE_MIN_VERSION "0.9.19")
+endif(NOT CLUCENE_MIN_VERSION)
+
+IF(EXISTS ${PROJECT_CMAKE}/CLuceneConfig.cmake)
+ INCLUDE(${PROJECT_CMAKE}/CLuceneConfig.cmake)
+ENDIF(EXISTS ${PROJECT_CMAKE}/CLuceneConfig.cmake)
+
+SET(TRIAL_LIBRARY_PATHS
+ $ENV{CLUCENE_HOME}/lib${LIB_SUFFIX}
+ ${CMAKE_INSTALL_PREFIX}/lib${LIB_SUFFIX}
+ /usr/local/lib${LIB_SUFFIX}
+ /usr/lib${LIB_SUFFIX}
+ /sw/lib${LIB_SUFFIX}
+ /usr/pkg/lib${LIB_SUFFIX}
+ )
+SET(TRIAL_INCLUDE_PATHS
+ $ENV{CLUCENE_HOME}/include
+ ${CMAKE_INSTALL_PREFIX}/include
+ /usr/local/include
+ /usr/include
+ /sw/include
+ /usr/pkg/include
+ )
+FIND_LIBRARY_WITH_DEBUG(CLUCENE_LIBRARY
+ WIN32_DEBUG_POSTFIX d
+ NAMES clucene clucene-core
+ PATHS ${TRIAL_LIBRARY_PATHS})
+IF (CLUCENE_LIBRARY)
+ MESSAGE(STATUS "Found CLucene library: ${CLUCENE_LIBRARY}")
+ENDIF (CLUCENE_LIBRARY)
+FIND_PATH(CLUCENE_INCLUDE_DIR
+ NAMES CLucene.h
+ PATHS ${TRIAL_INCLUDE_PATHS})
+
+IF (CLUCENE_INCLUDE_DIR)
+ MESSAGE(STATUS "Found CLucene include dir: ${CLUCENE_INCLUDE_DIR}")
+ENDIF (CLUCENE_INCLUDE_DIR)
+
+IF(WIN32)
+ SET(TRIAL_LIBRARY_PATHS ${CLUCENE_INCLUDE_DIR})
+ENDIF(WIN32)
+
+SET(CLUCENE_GOOD_VERSION TRUE)
+
+FIND_PATH(CLUCENE_LIBRARY_DIR
+ NAMES CLucene/clucene-config.h PATHS ${TRIAL_LIBRARY_PATHS} ${TRIAL_INCLUDE_PATHS} NO_DEFAULT_PATH)
+IF (CLUCENE_LIBRARY_DIR)
+ MESSAGE(STATUS "Found CLucene library dir: ${CLUCENE_LIBRARY_DIR}")
+ FILE(READ ${CLUCENE_LIBRARY_DIR}/CLucene/clucene-config.h CLCONTENT)
+ STRING(REGEX MATCH "_CL_VERSION +\".*\"" CLMATCH ${CLCONTENT})
+ IF (CLMATCH)
+ STRING(REGEX REPLACE "_CL_VERSION +\"(.*)\"" "\\1" CLUCENE_VERSION ${CLMATCH})
+ IF (CLUCENE_VERSION STRLESS "${CLUCENE_MIN_VERSION}")
+ MESSAGE(ERROR " CLucene version ${CLUCENE_VERSION} is less than the required minimum ${CLUCENE_MIN_VERSION}")
+ SET(CLUCENE_GOOD_VERSION FALSE)
+ ENDIF (CLUCENE_VERSION STRLESS "${CLUCENE_MIN_VERSION}")
+ IF (CLUCENE_VERSION STREQUAL "0.9.17")
+ MESSAGE(ERROR "CLucene version 0.9.17 is not supported.")
+ SET(CLUCENE_GOOD_VERSION FALSE)
+ ENDIF (CLUCENE_VERSION STREQUAL "0.9.17")
+ ENDIF (CLMATCH)
+ENDIF (CLUCENE_LIBRARY_DIR)
+
+IF(CLUCENE_INCLUDE_DIR AND CLUCENE_LIBRARY AND CLUCENE_LIBRARY_DIR AND CLUCENE_GOOD_VERSION)
+ SET(CLucene_FOUND TRUE)
+ENDIF(CLUCENE_INCLUDE_DIR AND CLUCENE_LIBRARY AND CLUCENE_LIBRARY_DIR AND CLUCENE_GOOD_VERSION)
+
+IF(CLucene_FOUND)
+ IF(NOT CLucene_FIND_QUIETLY)
+ MESSAGE(STATUS "Found CLucene: ${CLUCENE_LIBRARY} version ${CLUCENE_VERSION}")
+ ENDIF(NOT CLucene_FIND_QUIETLY)
+ELSE(CLucene_FOUND)
+ IF(CLucene_FIND_REQUIRED)
+ MESSAGE(FATAL_ERROR "Could not find CLucene.")
+ ENDIF(CLucene_FIND_REQUIRED)
+ENDIF(CLucene_FOUND)
+
+MARK_AS_ADVANCED(
+ CLUCENE_INCLUDE_DIR
+ CLUCENE_LIBRARY_DIR
+ CLUCENE_LIBRARY
+ )
diff --git a/README b/README
index fccf7aef1..f58b0449c 100644
--- a/README
+++ b/README
@@ -1,9 +1,9 @@
Quickstart on Ubuntu
--------------------
- $ sudo apt-get install build-essential cmake libtag1c2a libtag1-dev liblastfm-dev \
- libqt4-dev libqt4-sql-sqlite libvorbis-dev libmad0-dev libflac++-dev \
- libasound2-dev libboost-dev zlib1g-dev libgnutls-dev pkg-config
+ $ sudo apt-get install build-essential cmake libtag1c2a libtag1-dev liblastfm-dev libqt4-dev \
+ libqt4-sql-sqlite libvorbis-dev libmad0-dev libflac++-dev libasound2-dev \
+ libboost-dev zlib1g-dev libgnutls-dev libclucene-dev pkg-config
Gloox 1.0 (XMPP library)
@@ -46,7 +46,7 @@ Quickstart on OS X
Install homebrew
$ ruby -e "$(curl -fsSL https://gist.github.com/raw/323731/install_homebrew.rb)"
- $ brew install qt qjson gloox libmad libvorbis flac taglib boost liblastfm
+ $ brew install qt qjson gloox libmad libvorbis flac taglib boost liblastfm clucene
Install libEchnoest as per the above instructions.
@@ -78,6 +78,7 @@ Dependencies
SQLite 3.6.22 http://www.sqlite.org/
TagLib 1.6.2 http://developer.kde.org/~wheeler/taglib.html
Boost 1.3x http://www.boost.org/
+ CLucene 0.9.21 http://clucene.sourceforge.net/
libmad 0.15.1b http://www.underbit.com/products/mad/
libvorbis 1.2.3 http://xiph.org/vorbis/
libogg 1.1.4 http://xiph.org/ogg/
diff --git a/src/libtomahawk/CMakeLists.txt b/src/libtomahawk/CMakeLists.txt
index 866754081..90cde3b93 100644
--- a/src/libtomahawk/CMakeLists.txt
+++ b/src/libtomahawk/CMakeLists.txt
@@ -111,7 +111,6 @@ set( libSources
playlist/dynamic/widgets/MiscControlWidgets.cpp
playlist/dynamic/widgets/CollapsibleControls.cpp
-
network/bufferiodevice.cpp
network/msgprocessor.cpp
network/filetransferconnection.cpp
@@ -277,6 +276,8 @@ include_directories( . ${CMAKE_CURRENT_BINARY_DIR} ..
${QT_INCLUDE_DIR}
${LIBECHONEST_INCLUDE_DIR}
${LIBECHONEST_INCLUDE_DIR}/..
+ ${CLUCENE_INCLUDE_DIR}
+ ${CLUCENE_LIBRARY_DIR}
../../libportfwd/include
../../include
../network
@@ -347,6 +348,7 @@ target_link_libraries( tomahawklib
ogg
FLAC++
jdns
+ clucene
)
install( TARGETS tomahawklib DESTINATION lib )
diff --git a/src/libtomahawk/database/database.h b/src/libtomahawk/database/database.h
index 156ba6743..f088b9648 100644
--- a/src/libtomahawk/database/database.h
+++ b/src/libtomahawk/database/database.h
@@ -16,10 +16,9 @@
HOWEVER, we're using the command pattern to serialize access to the database
and provide an async api. You create a DatabaseCommand object, and add it to
- the queue of work. There is a single thread responsible for exec'ing all
- the commands, so sqlite only does one thing at a time.
-
- Update: 1 thread for mutates, one for readonly queries.
+ the queue of work. There is a threadpool responsible for exec'ing all
+ the non-mutating (readonly) commands and one separate thread for mutating ones,
+ so sqlite doesn't write to the Database from multiple threads.
*/
class DLLEXPORT Database : public QObject
{
diff --git a/src/libtomahawk/database/databasecommand_addfiles.cpp b/src/libtomahawk/database/databasecommand_addfiles.cpp
index 0426d8638..1f9485f87 100644
--- a/src/libtomahawk/database/databasecommand_addfiles.cpp
+++ b/src/libtomahawk/database/databasecommand_addfiles.cpp
@@ -16,7 +16,7 @@ QVariantList
DatabaseCommand_AddFiles::files() const
{
QVariantList list;
- foreach( const QVariant& v, m_files )
+ foreach ( const QVariant& v, m_files )
{
// replace url with the id, we don't leak file paths over the network.
QVariantMap m = v.toMap();
@@ -73,11 +73,7 @@ DatabaseCommand_AddFiles::exec( DatabaseImpl* dbi )
query_trackattr.prepare( "INSERT INTO track_attributes(id, k, v) "
"VALUES (?,?,?)" );
query_file_del.prepare( QString( "DELETE FROM file WHERE source %1 AND url = ?" )
- .arg( source()->isLocal() ? "IS NULL" : QString( "= %1" ).arg( source()->id() )
- ) );
-
- int maxart, maxalb, maxtrk; // store max id, so we can index new ones after
- maxart = maxalb = maxtrk = 0;
+ .arg( source()->isLocal() ? "IS NULL" : QString( "= %1" ).arg( source()->id() ) ) );
int added = 0;
QVariant srcid = source()->isLocal() ?
@@ -126,29 +122,28 @@ DatabaseCommand_AddFiles::exec( DatabaseImpl* dbi )
}
else
{
- if( added % 100 == 0 ) qDebug() << "Inserted" << added;
+ if( added % 100 == 0 )
+ qDebug() << "Inserted" << added;
}
// get internal IDs for art/alb/trk
fileid = query_file.lastInsertId().toInt();
// insert the new fileid, set the url for our use:
m.insert( "id", fileid );
- if( !source()->isLocal() ) m["url"] = QString( "servent://%1\t%2" )
- .arg( source()->userName() )
- .arg( fileid );
+ if( !source()->isLocal() )
+ m["url"] = QString( "servent://%1\t%2" )
+ .arg( source()->userName() )
+ .arg( fileid );
v = m;
bool isnew;
int artid = dbi->artistId( artist, isnew );
if( artid < 1 ) continue;
- if( isnew && maxart == 0 ) maxart = artid;
int trkid = dbi->trackId( artid, track, isnew );
if( trkid < 1 ) continue;
- if( isnew && maxtrk == 0 ) maxtrk = trkid;
int albid = dbi->albumId( artid, album, isnew );
- if( albid > 0 && isnew && maxalb == 0 ) maxalb = albid;
// Now add the association
query_filejoin.bindValue( 0, fileid );
@@ -172,11 +167,8 @@ DatabaseCommand_AddFiles::exec( DatabaseImpl* dbi )
qDebug() << "Inserted" << added;
// TODO building the index could be a separate job, outside this transaction
- if(maxart) dbi->updateSearchIndex( "artist", maxart );
- if(maxalb) dbi->updateSearchIndex( "album", maxalb );
- if(maxtrk) dbi->updateSearchIndex( "track", maxtrk );
+ dbi->updateSearchIndex();
qDebug() << "Committing" << added << "tracks...";
- qDebug() << "Done.";
emit done( m_files, source()->collection() );
}
diff --git a/src/libtomahawk/database/databasecommand_alltracks.cpp b/src/libtomahawk/database/databasecommand_alltracks.cpp
index f3bf6812d..13b721c0d 100644
--- a/src/libtomahawk/database/databasecommand_alltracks.cpp
+++ b/src/libtomahawk/database/databasecommand_alltracks.cpp
@@ -89,8 +89,8 @@ DatabaseCommand_AllTracks::exec( DatabaseImpl* dbi )
attr[ attrQuery.value( 0 ).toString() ] = attrQuery.value( 1 ).toString();
}
- Tomahawk::query_ptr query = Tomahawk::query_ptr( new Tomahawk::Query( t ) );
- t.insert( "score", 1.0);
+ Tomahawk::query_ptr query = Tomahawk::Query::get( t, false );
+ t["score"] = 1.0;
Tomahawk::result_ptr result = Tomahawk::result_ptr( new Tomahawk::Result( t, m_collection ) );
result->setAttributes( attr );
diff --git a/src/libtomahawk/database/databasecommand_loadplaylistentries.cpp b/src/libtomahawk/database/databasecommand_loadplaylistentries.cpp
index 722878e57..05fc5cd11 100644
--- a/src/libtomahawk/database/databasecommand_loadplaylistentries.cpp
+++ b/src/libtomahawk/database/databasecommand_loadplaylistentries.cpp
@@ -62,8 +62,8 @@ void DatabaseCommand_LoadPlaylistEntries::generateEntries( DatabaseImpl* dbi )
m.insert( "track", query.value( 1 ).toString() );
m.insert( "resulthint", query.value( 8 ).toString() );
m.insert( "qid", uuid() );
-
- Tomahawk::query_ptr q( new Tomahawk::Query( m ) );
+
+ Tomahawk::query_ptr q = Tomahawk::Query::get( m, false );
e->setQuery( q );
m_entrymap.insert( e->guid(), e );
diff --git a/src/libtomahawk/database/databasecommand_logplayback.cpp b/src/libtomahawk/database/databasecommand_logplayback.cpp
index 1aa617859..5ce6976a8 100644
--- a/src/libtomahawk/database/databasecommand_logplayback.cpp
+++ b/src/libtomahawk/database/databasecommand_logplayback.cpp
@@ -5,7 +5,6 @@
#include "collection.h"
#include "database/database.h"
#include "databaseimpl.h"
-#include "pipeline.h"
#include "network/servent.h"
using namespace Tomahawk;
@@ -26,9 +25,7 @@ DatabaseCommand_LogPlayback::postCommitHook()
m.insert( "track", m_track );
m.insert( "artist", m_artist );
m.insert( "qid", uuid() );
- Tomahawk::query_ptr q( new Tomahawk::Query( m ) );
-
- Tomahawk::Pipeline::instance()->add( q );
+ Tomahawk::query_ptr q = Tomahawk::Query::get( m );
if ( m_action == Finished )
{
diff --git a/src/libtomahawk/database/databasecommand_playbackhistory.cpp b/src/libtomahawk/database/databasecommand_playbackhistory.cpp
index d6727a4e8..e7f7b8979 100644
--- a/src/libtomahawk/database/databasecommand_playbackhistory.cpp
+++ b/src/libtomahawk/database/databasecommand_playbackhistory.cpp
@@ -3,7 +3,7 @@
#include <QSqlQuery>
#include "databaseimpl.h"
-#include "pipeline.h"
+
void
DatabaseCommand_PlaybackHistory::exec( DatabaseImpl* dbi )
@@ -49,7 +49,7 @@ DatabaseCommand_PlaybackHistory::exec( DatabaseImpl* dbi )
m.insert( "artist", query_track.value( 1 ).toString() );
m.insert( "qid", uuid() );
- Tomahawk::query_ptr q( new Tomahawk::Query( m ) );
+ Tomahawk::query_ptr q = Tomahawk::Query::get( m );
ql << q;
}
}
@@ -57,8 +57,5 @@ DatabaseCommand_PlaybackHistory::exec( DatabaseImpl* dbi )
qDebug() << Q_FUNC_INFO << ql.length();
if ( ql.count() )
- {
- Tomahawk::Pipeline::instance()->add( ql );
emit tracks( ql );
- }
}
diff --git a/src/libtomahawk/database/databasecommand_resolve.cpp b/src/libtomahawk/database/databasecommand_resolve.cpp
index 20b78dab5..7cf8ac5cd 100644
--- a/src/libtomahawk/database/databasecommand_resolve.cpp
+++ b/src/libtomahawk/database/databasecommand_resolve.cpp
@@ -8,10 +8,9 @@
using namespace Tomahawk;
-DatabaseCommand_Resolve::DatabaseCommand_Resolve( const QVariant& v, bool searchlocal )
+DatabaseCommand_Resolve::DatabaseCommand_Resolve( const QVariant& v )
: DatabaseCommand()
, m_v( v )
- , m_searchlocal( searchlocal )
{
}
@@ -72,6 +71,7 @@ DatabaseCommand_Resolve::exec( DatabaseImpl* lib )
if( artists.length() == 0 || tracks.length() == 0 )
{
//qDebug() << "No candidates found in first pass, aborting resolve" << artistname << trackname;
+ emit results( qid, res );
return;
}
@@ -98,11 +98,9 @@ DatabaseCommand_Resolve::exec( DatabaseImpl* lib )
"WHERE "
"artist.id = file_join.artist AND "
"track.id = file_join.track AND "
- "file.source %1 AND "
"file.id = file_join.file AND "
- "file_join.artist IN (%2) AND "
- "file_join.track IN (%3)"
- ).arg( m_searchlocal ? "IS NULL" : " > 0 " )
+ "file_join.artist IN (%1) AND "
+ "file_join.track IN (%2)" )
.arg( artsl.join( "," ) )
.arg( trksl.join( "," ) );
@@ -130,7 +128,7 @@ DatabaseCommand_Resolve::exec( DatabaseImpl* lib )
source_ptr s;
const QString url_str = files_query.value( 0 ).toString();
- if( m_searchlocal )
+ if( files_query.value( 13 ).toUInt() == 0 )
{
s = SourceList::instance()->getLocal();
m["url"] = url_str;
diff --git a/src/libtomahawk/database/databasecommand_resolve.h b/src/libtomahawk/database/databasecommand_resolve.h
index 78015f920..e62ce076a 100644
--- a/src/libtomahawk/database/databasecommand_resolve.h
+++ b/src/libtomahawk/database/databasecommand_resolve.h
@@ -13,7 +13,7 @@ class DLLEXPORT DatabaseCommand_Resolve : public DatabaseCommand
{
Q_OBJECT
public:
- explicit DatabaseCommand_Resolve( const QVariant& v, bool searchlocal );
+ explicit DatabaseCommand_Resolve( const QVariant& v );
virtual QString commandname() const { return "dbresolve"; }
virtual bool doesMutates() const { return false; }
@@ -27,7 +27,6 @@ public slots:
private:
QVariant m_v;
- bool m_searchlocal;
float how_similar( const QVariantMap& q, const QVariantMap& r );
static int levenshtein( const QString& source, const QString& target );
diff --git a/src/libtomahawk/database/databasecommand_updatesearchindex.cpp b/src/libtomahawk/database/databasecommand_updatesearchindex.cpp
index f4c517ffb..67d9e9b60 100644
--- a/src/libtomahawk/database/databasecommand_updatesearchindex.cpp
+++ b/src/libtomahawk/database/databasecommand_updatesearchindex.cpp
@@ -1,117 +1,39 @@
#include "databasecommand_updatesearchindex.h"
-DatabaseCommand_UpdateSearchIndex::DatabaseCommand_UpdateSearchIndex( const QString& t, int p )
+DatabaseCommand_UpdateSearchIndex::DatabaseCommand_UpdateSearchIndex()
: DatabaseCommand()
- , table( t )
- , pkey( p )
{
- if( table != "artist" && table != "track" && table != "album" )
- {
- Q_ASSERT(false);
- return;
- }
}
-void DatabaseCommand_UpdateSearchIndex::exec(DatabaseImpl *db)
+void
+DatabaseCommand_UpdateSearchIndex::indexTable( DatabaseImpl* db, const QString& table )
{
qDebug() << Q_FUNC_INFO;
-
- if( table != "artist" && table != "track" && table != "album" )
- {
- Q_ASSERT(false);
- return;
- }
-
- // if pkey is 0, consult DB to see what needs indexing
- if( pkey == 0 )
- {
- TomahawkSqlQuery q = db->newquery();
- q.exec( QString("SELECT coalesce(max(id),0) from %1_search_index").arg(table) );
- q.next();
- pkey = 1 + q.value(0).toInt();
- qDebug() << "updateSearchIndex" << table << "consulted DB, starting at" << pkey;
- }
-
+
TomahawkSqlQuery query = db->newquery();
- qDebug() << "Building index for" << table << ">= id" << pkey;
- QString searchtable( table + "_search_index" );
- query.exec(QString( "SELECT id, sortname FROM %1 WHERE id >= %2" ).arg( table ).arg(pkey ) );
-
- TomahawkSqlQuery upq = db->newquery();
- TomahawkSqlQuery inq = db->newquery();
- inq.prepare( "INSERT INTO "+ searchtable +" (ngram, id, num) VALUES (?,?,?)" );
- upq.prepare( "UPDATE "+ searchtable +" SET num=num+? WHERE ngram=? AND id=?" );
-
- int num_names = 0;
- int num_ngrams = 0;
- int id;
- QString name;
- QMap<QString, int> ngrammap;
-
- // this is the new ngram map we build up, to be merged into the
- // main one in FuzzyIndex:
- QHash< QString, QMap<quint32, quint16> > idx;
-
- while( query.next() )
+ qDebug() << "Building index for" << table;
+ query.exec( QString( "SELECT id, name FROM %1" ).arg( table ) );
+
+ QMap< unsigned int, QString > fields;
+ while ( query.next() )
{
- id = query.value( 0 ).toInt();
- name = query.value( 1 ).toString();
- num_names++;
- inq.bindValue( 1, id ); // set id
- upq.bindValue( 2, id ); // set id
- ngrammap = DatabaseImpl::ngrams( name );
- QMapIterator<QString, int> i( ngrammap );
-
- while ( i.hasNext() )
- {
- i.next();
- num_ngrams++;
- upq.bindValue( 0, i.value() ); //num
- upq.bindValue( 1, i.key() ); // ngram
- upq.exec();
-
- if( upq.numRowsAffected() == 0 )
- {
- inq.bindValue( 0, i.key() ); //ngram
- inq.bindValue( 2, i.value() ); //num
- inq.exec();
- if( inq.numRowsAffected() == 0 )
- {
- qDebug() << "Error updating search index:" << id << name;
- continue;
- }
- }
-
- // update ngram cache:
- QMapIterator<QString, int> iter( ngrammap );
- while ( iter.hasNext() )
- {
- iter.next();
- if( idx.contains( iter.key() ) )
- {
- idx[ iter.key() ][ id ] += iter.value();
- }
- else
- {
- QMap<quint32, quint16> tmp;
- tmp.insert( id, iter.value() );
- idx.insert( iter.key(), tmp );
- }
- }
-
-
- }
+ fields.insert( query.value( 0 ).toUInt(), query.value( 1 ).toString() );
}
-
- // merge in our ngrams into the main index
- QMetaObject::invokeMethod( &(db->m_fuzzyIndex),
- "mergeIndex",
- Qt::QueuedConnection,
- Q_ARG( QString, table ),
- QGenericArgument( "QHash< QString, QMap<quint32, quint16> >", &idx )
- );
-
- qDebug() << "Finished indexing" << num_names <<" names," << num_ngrams << "ngrams.";
+
+ db->m_fuzzyIndex->appendFields( table, fields );
+}
+
+
+void
+DatabaseCommand_UpdateSearchIndex::exec( DatabaseImpl* db )
+{
+ db->m_fuzzyIndex->beginIndexing();
+
+ indexTable( db, "artist" );
+ indexTable( db, "album" );
+ indexTable( db, "track" );
+
+ db->m_fuzzyIndex->endIndexing();
}
diff --git a/src/libtomahawk/database/databasecommand_updatesearchindex.h b/src/libtomahawk/database/databasecommand_updatesearchindex.h
index c9278e0cb..d8d74ed02 100644
--- a/src/libtomahawk/database/databasecommand_updatesearchindex.h
+++ b/src/libtomahawk/database/databasecommand_updatesearchindex.h
@@ -10,21 +10,19 @@ class DLLEXPORT DatabaseCommand_UpdateSearchIndex : public DatabaseCommand
{
Q_OBJECT
public:
- explicit DatabaseCommand_UpdateSearchIndex(const QString& table, int pkey);
+ explicit DatabaseCommand_UpdateSearchIndex();
virtual QString commandname() const { return "updatesearchindex"; }
virtual bool doesMutates() const { return true; }
- virtual void exec(DatabaseImpl* db);
+ virtual void exec( DatabaseImpl* db );
signals:
void indexUpdated();
-public slots:
-
private:
+ void indexTable( DatabaseImpl* db, const QString& table );
+
QString table;
- int pkey;
-
};
#endif // DATABASECOMMAND_UPDATESEARCHINDEX_H
diff --git a/src/libtomahawk/database/databaseimpl.cpp b/src/libtomahawk/database/databaseimpl.cpp
index 3c94603b6..fe38d63f0 100644
--- a/src/libtomahawk/database/databaseimpl.cpp
+++ b/src/libtomahawk/database/databaseimpl.cpp
@@ -16,16 +16,15 @@
*/
#include "schema.sql.h"
-#define CURRENT_SCHEMA_VERSION 17
+#define CURRENT_SCHEMA_VERSION 19
DatabaseImpl::DatabaseImpl( const QString& dbname, Database* parent )
: QObject( (QObject*) parent )
, m_lastartid( 0 )
, m_lastalbid( 0 )
, m_lasttrkid( 0 )
- , m_fuzzyIndex( *this )
{
- connect( this, SIGNAL(indexReady()), parent, SIGNAL(indexReady()) );
+ connect( this, SIGNAL( indexReady() ), parent, SIGNAL( indexReady() ) );
db = QSqlDatabase::addDatabase( "QSQLITE", "tomahawk" );
db.setDatabaseName( dbname );
@@ -95,6 +94,8 @@ DatabaseImpl::DatabaseImpl( const QString& dbname, Database* parent )
// in case of unclean shutdown last time:
query.exec( "UPDATE source SET isonline = 'false'" );
+
+ m_fuzzyIndex = new FuzzyIndex( *this );
}
@@ -102,6 +103,8 @@ DatabaseImpl::~DatabaseImpl()
{
m_indexThread.quit();
m_indexThread.wait( 5000 );
+
+ delete m_fuzzyIndex;
}
@@ -109,17 +112,17 @@ void
DatabaseImpl::loadIndex()
{
// load ngram index in the background
- m_fuzzyIndex.moveToThread( &m_indexThread );
- connect( &m_indexThread, SIGNAL(started()), &m_fuzzyIndex, SLOT(loadNgramIndex()) );
- connect( &m_fuzzyIndex, SIGNAL(indexReady()), this, SIGNAL(indexReady()) );
+ m_fuzzyIndex->moveToThread( &m_indexThread );
+ connect( &m_indexThread, SIGNAL( started() ), m_fuzzyIndex, SLOT( loadLuceneIndex() ) );
+ connect( m_fuzzyIndex, SIGNAL( indexReady() ), this, SIGNAL( indexReady() ) );
m_indexThread.start();
}
void
-DatabaseImpl::updateSearchIndex( const QString& table, int pkey )
+DatabaseImpl::updateSearchIndex()
{
- DatabaseCommand* cmd = new DatabaseCommand_UpdateSearchIndex(table, pkey);
+ DatabaseCommand* cmd = new DatabaseCommand_UpdateSearchIndex();
Database::instance()->enqueue( QSharedPointer<DatabaseCommand>( cmd ) );
}
@@ -323,41 +326,24 @@ DatabaseImpl::albumId( int artistid, const QString& name_orig, bool& isnew )
QList< int >
-DatabaseImpl::searchTable( const QString& table, const QString& name_orig, uint limit )
+DatabaseImpl::searchTable( const QString& table, const QString& name, uint limit )
{
QList< int > results;
if( table != "artist" && table != "track" && table != "album" )
return results;
- QString name = sortname( name_orig );
+ QMap< int, float > resultsmap = m_fuzzyIndex->search( table, name );
- // first check for exact matches:
- TomahawkSqlQuery query = newquery();
- query.prepare( QString( "SELECT id FROM %1 WHERE sortname = ?" ).arg( table ) );
- query.addBindValue( name );
- query.exec();
-
- while( query.next() )
- results.append( query.value( 0 ).toInt() );
-
- // ngram stuff only works on tracks over a certain length atm:
- if( name_orig.length() > 3 )
+ QList< QPair<int, float> > resultslist;
+ foreach( int i, resultsmap.keys() )
{
- // consult ngram index to find candidates:
- QMap< int, float > resultsmap = m_fuzzyIndex.search( table, name );
+ resultslist << QPair<int, float>( i, (float)resultsmap.value( i ) );
+ }
+ qSort( resultslist.begin(), resultslist.end(), DatabaseImpl::scorepairSorter );
- //qDebug() << "results map for" << table << resultsmap.size();
- QList< QPair<int,float> > resultslist;
- foreach( int i, resultsmap.keys() )
- {
- resultslist << QPair<int,float>( i, (float)resultsmap.value( i ) );
- }
- qSort( resultslist.begin(), resultslist.end(), DatabaseImpl::scorepairSorter );
-
- for( int k = 0; k < resultslist.size() && k < (int)limit; ++k )
- {
- results << resultslist.at( k ).first;
- }
+ for( int k = 0; k < resultslist.count(); k++ )
+ {
+ results << resultslist.at( k ).first;
}
return results;
@@ -382,35 +368,6 @@ DatabaseImpl::getTrackFids( int tid )
}
-QMap< QString, int >
-DatabaseImpl::ngrams( const QString& str_orig )
-{
- static QMap< QString, QMap<QString, int> > memo;
- if( memo.contains( str_orig ) )
- return memo.value( str_orig );
-
- int n = 3;
- QMap<QString, int> ret;
- QString str( " " + DatabaseImpl::sortname( str_orig ) + " " );
- int num = str.length();
- QString ngram;
-
- for( int j = 0; j < num - ( n - 1 ); j++ )
- {
- ngram = str.mid( j, n );
- Q_ASSERT( ngram.length() == n );
-
- if( ret.contains( ngram ) )
- ret[ngram]++;
- else
- ret[ngram] = 1;
- }
-
- memo.insert( str_orig, ret );
- return ret;
-}
-
-
QString
DatabaseImpl::sortname( const QString& str )
{
diff --git a/src/libtomahawk/database/databaseimpl.h b/src/libtomahawk/database/databaseimpl.h
index 32ec94ad4..7fcd3f299 100644
--- a/src/libtomahawk/database/databaseimpl.h
+++ b/src/libtomahawk/database/databaseimpl.h
@@ -36,10 +36,9 @@ public:
int trackId( int artistid, const QString& name_orig, bool& isnew );
int albumId( int artistid, const QString& name_orig, bool& isnew );
- QList< int > searchTable( const QString& table, const QString& name_orig, uint limit = 10 );
+ QList< int > searchTable( const QString& table, const QString& name, uint limit = 10 );
QList< int > getTrackFids( int tid );
- static QMap<QString,int> ngrams( const QString& str_orig );
static QString sortname( const QString& str );
QVariantMap artist( int id );
@@ -54,7 +53,7 @@ public:
}
// indexes entries from "table" where id >= pkey
- void updateSearchIndex( const QString& table, int pkey );
+ void updateSearchIndex();
const QString& dbid() const { return m_dbid; }
@@ -76,7 +75,7 @@ private:
QString m_dbid;
QThread m_indexThread;
- FuzzyIndex m_fuzzyIndex;
+ FuzzyIndex* m_fuzzyIndex;
};
#endif // DATABASEIMPL_H
diff --git a/src/libtomahawk/database/databaseresolver.cpp b/src/libtomahawk/database/databaseresolver.cpp
index 60326949f..639a3fede 100644
--- a/src/libtomahawk/database/databaseresolver.cpp
+++ b/src/libtomahawk/database/databaseresolver.cpp
@@ -4,9 +4,8 @@
#include "database/database.h"
#include "database/databasecommand_resolve.h"
-DatabaseResolver::DatabaseResolver( bool searchlocal, int weight )
+DatabaseResolver::DatabaseResolver( int weight )
: Resolver()
- , m_searchlocal( searchlocal )
, m_weight( weight )
{
}
@@ -17,13 +16,7 @@ DatabaseResolver::resolve( const QVariant& v )
{
//qDebug() << Q_FUNC_INFO << v;
- if( !m_searchlocal )
- {
- if( Servent::instance()->numConnectedPeers() == 0 )
- return;
- }
-
- DatabaseCommand_Resolve* cmd = new DatabaseCommand_Resolve( v, m_searchlocal );
+ DatabaseCommand_Resolve* cmd = new DatabaseCommand_Resolve( v );
connect( cmd, SIGNAL( results( Tomahawk::QID, QList< Tomahawk::result_ptr> ) ),
SLOT( gotResults( Tomahawk::QID, QList< Tomahawk::result_ptr> ) ), Qt::QueuedConnection );
@@ -45,5 +38,5 @@ DatabaseResolver::gotResults( const Tomahawk::QID qid, QList< Tomahawk::result_p
QString
DatabaseResolver::name() const
{
- return QString( "Database (%1)" ).arg( m_searchlocal ? "local" : "remote" );
+ return QString( "DatabaseResolver" );
}
diff --git a/src/libtomahawk/database/databaseresolver.h b/src/libtomahawk/database/databaseresolver.h
index b40f2a200..81bfca9d4 100644
--- a/src/libtomahawk/database/databaseresolver.h
+++ b/src/libtomahawk/database/databaseresolver.h
@@ -13,7 +13,7 @@ class DLLEXPORT DatabaseResolver : public Tomahawk::Resolver
Q_OBJECT
public:
- explicit DatabaseResolver( bool searchlocal, int weight );
+ explicit DatabaseResolver( int weight );
virtual QString name() const;
virtual unsigned int weight() const { return m_weight; }
@@ -26,7 +26,6 @@ private slots:
void gotResults( const Tomahawk::QID qid, QList< Tomahawk::result_ptr> results );
private:
- bool m_searchlocal;
int m_weight;
};
diff --git a/src/libtomahawk/database/databaseworker.cpp b/src/libtomahawk/database/databaseworker.cpp
index 7489ee615..510d53064 100644
--- a/src/libtomahawk/database/databaseworker.cpp
+++ b/src/libtomahawk/database/databaseworker.cpp
@@ -40,9 +40,10 @@ DatabaseWorker::run()
void
DatabaseWorker::enqueue( const QSharedPointer<DatabaseCommand>& cmd )
{
+ m_outstanding++;
+
QMutexLocker lock( &m_mut );
m_commands << cmd;
- m_outstanding++;
if ( m_outstanding == 1 )
QTimer::singleShot( 0, this, SLOT( doWork() ) );
@@ -163,11 +164,7 @@ DatabaseWorker::doWork()
cmd->emitFinished();
- {
- QMutexLocker lock( &m_mut );
- m_outstanding--;
- }
-
+ m_outstanding--;
if ( m_outstanding > 0 )
QTimer::singleShot( 0, this, SLOT( doWork() ) );
}
diff --git a/src/libtomahawk/database/fuzzyindex.cpp b/src/libtomahawk/database/fuzzyindex.cpp
index a4dbf5478..4ffdd7b9a 100644
--- a/src/libtomahawk/database/fuzzyindex.cpp
+++ b/src/libtomahawk/database/fuzzyindex.cpp
@@ -1,139 +1,147 @@
#include "fuzzyindex.h"
#include "databaseimpl.h"
+#include "utils/tomahawkutils.h"
+#include <QDir>
#include <QTime>
+#include <CLucene.h>
-FuzzyIndex::FuzzyIndex( DatabaseImpl &db )
+
+FuzzyIndex::FuzzyIndex( DatabaseImpl& db )
: QObject()
, m_db( db )
- , m_loaded( false )
+ , m_luceneReader( 0 )
+ , m_luceneSearcher( 0 )
{
+ QString lucenePath = TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" );
+ bool create = !lucene::index::IndexReader::indexExists( lucenePath.toStdString().c_str() );
+ m_luceneDir = lucene::store::FSDirectory::getDirectory( lucenePath.toStdString().c_str(), create );
+
+ m_analyzer = new lucene::analysis::SimpleAnalyzer();
+}
+
+
+FuzzyIndex::~FuzzyIndex()
+{
+ delete m_luceneSearcher;
+ delete m_luceneReader;
+ delete m_analyzer;
+ delete m_luceneDir;
}
void
-FuzzyIndex::loadNgramIndex()
+FuzzyIndex::beginIndexing()
{
- // this updates the index in the DB, if needed:
- qDebug() << "Checking catalogue is fully indexed...";
- m_db.updateSearchIndex( "artist", 0 );
- m_db.updateSearchIndex( "album", 0 );
- m_db.updateSearchIndex( "track", 0 );
+ m_mutex.lock();
+ lucene::index::IndexWriter luceneWriter = lucene::index::IndexWriter( m_luceneDir, m_analyzer, true );
+}
- // loads index from DB into memory:
- qDebug() << "Loading search index for catalogue metadata..." << thread();
- loadNgramIndex_helper( m_artist_ngrams, "artist" );
- loadNgramIndex_helper( m_album_ngrams, "album" );
- loadNgramIndex_helper( m_track_ngrams, "track" );
- m_loaded = true;
+
+void
+FuzzyIndex::endIndexing()
+{
+ m_mutex.unlock();
emit indexReady();
}
void
-FuzzyIndex::loadNgramIndex_helper( QHash< QString, QMap<quint32, quint16> >& idx, const QString& table, unsigned int fromkey )
+FuzzyIndex::appendFields( const QString& table, const QMap< unsigned int, QString >& fields )
{
- QTime t;
- t.start();
-
- TomahawkSqlQuery query = m_db.newquery();
- query.exec( QString( "SELECT ngram, id, num "
- "FROM %1_search_index "
- "WHERE id >= %2 "
- "ORDER BY ngram" ).arg( table ).arg( fromkey ) );
-
- QMap<quint32, quint16> ngram_idx;
- QString lastngram;
- while( query.next() )
+ delete m_luceneSearcher;
+ delete m_luceneReader;
+ m_luceneSearcher = 0;
+ m_luceneReader = 0;
+
+ bool create = !lucene::index::IndexReader::indexExists( TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" ).toStdString().c_str() );
+ lucene::index::IndexWriter luceneWriter = lucene::index::IndexWriter( m_luceneDir, m_analyzer, create );
+ lucene::document::Document doc;
+
+ QMapIterator< unsigned int, QString > it( fields );
+ while ( it.hasNext() )
{
- const QString ng = query.value( 0 ).toString();
- if( lastngram.isEmpty() )
- lastngram = ng;
+ it.next();
+ unsigned int id = it.key();
+ QString name = it.value();
- if( ng != lastngram )
{
- idx.insert( lastngram, ngram_idx );
- lastngram = ng;
- ngram_idx.clear();
+ lucene::document::Field* field = new lucene::document::Field( table.toStdWString().c_str(), name.toStdWString().c_str(),
+ lucene::document::Field::STORE_YES | lucene::document::Field::INDEX_UNTOKENIZED );
+ doc.add( *field );
}
-
- ngram_idx.insert( query.value( 1 ).toUInt(),
- query.value( 2 ).toUInt() );
+
+ {
+ lucene::document::Field* field = new lucene::document::Field( _T( "id" ), QString::number( id ).toStdWString().c_str(),
+ lucene::document::Field::STORE_YES | lucene::document::Field::INDEX_NO );
+ doc.add( *field );
+ }
+
+ luceneWriter.addDocument( &doc );
+ doc.clear();
}
-
- idx.insert( lastngram, ngram_idx );
- qDebug() << "Loaded" << idx.size()
- << "ngram entries for" << table
- << "in" << t.elapsed() << "ms";
+
+ luceneWriter.close();
}
-void FuzzyIndex::mergeIndex( const QString& table, const QHash< QString, QMap<quint32, quint16> >& tomerge )
+void
+FuzzyIndex::loadLuceneIndex()
{
- qDebug() << Q_FUNC_INFO << table << tomerge.keys().size();
-
- QHash< QString, QMap<quint32, quint16> >* idx;
- if ( table == "artist" ) idx = &m_artist_ngrams;
- else if( table == "album" ) idx = &m_album_ngrams;
- else if( table == "track" ) idx = &m_track_ngrams;
- else Q_ASSERT( false );
-
- if( tomerge.size() == 0 )
- return;
-
- if( idx->size() == 0 )
- {
- *idx = tomerge;
- }
- else
- {
- QList<QString> tmk = tomerge.keys();
- foreach( const QString& ngram, tmk )
- {
- if( idx->contains( ngram ) )
- {
- QList<unsigned int> tmkn = tomerge[ngram].keys();
- foreach( quint32 id, tmkn )
- {
- (*idx)[ ngram ][ id ] += tomerge[ngram][id];
- }
- }
- else
- {
- idx->insert( ngram, tomerge[ngram] );
- }
- }
- }
-
- qDebug() << Q_FUNC_INFO << table << "merge complete, num items:" << tomerge.size();
+ emit indexReady();
}
QMap< int, float >
FuzzyIndex::search( const QString& table, const QString& name )
{
+ QMutexLocker lock( &m_mutex );
+
QMap< int, float > resultsmap;
-
- QHash< QString, QMap<quint32, quint16> >* idx;
- if( table == "artist" ) idx = &m_artist_ngrams;
- else if( table == "album" ) idx = &m_album_ngrams;
- else if( table == "track" ) idx = &m_track_ngrams;
-
- QMap<QString,int> ngramsmap = DatabaseImpl::ngrams( name );
- foreach( const QString& ngram, ngramsmap.keys() )
+ if ( !m_luceneReader )
{
- if( !idx->contains( ngram ) )
- continue;
-
- //qDebug() << name_orig << "NGRAM:" << ngram << "candidates:" << (*idx)[ngram].size();
- QMapIterator<quint32, quint16> iter( (*idx)[ngram] );
- while( iter.hasNext() )
+ if ( !lucene::index::IndexReader::indexExists( TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" ).toStdString().c_str() ) )
{
- iter.next();
- resultsmap[ (int) iter.key() ] += (float) iter.value();
+ qDebug() << Q_FUNC_INFO << "index didn't exist.";
+ return resultsmap;
+ }
+
+ m_luceneReader = lucene::index::IndexReader::open( m_luceneDir );
+ m_luceneSearcher = new lucene::search::IndexSearcher( m_luceneReader );
+ }
+
+ if ( name.isEmpty() )
+ return resultsmap;
+
+ lucene::analysis::SimpleAnalyzer analyzer;
+ lucene::queryParser::QueryParser parser( table.toStdWString().c_str(), m_analyzer );
+ lucene::search::Hits* hits = 0;
+
+ lucene::search::FuzzyQuery* qry = new lucene::search::FuzzyQuery( new lucene::index::Term( table.toStdWString().c_str(),
+ name.toStdWString().c_str() ) );
+ hits = m_luceneSearcher->search( qry );
+
+ for ( unsigned int i = 0; i < hits->length(); i++ )
+ {
+ lucene::document::Document* d = &hits->doc( i );
+
+ float score = hits->score( i );
+ int id = QString::fromWCharArray( d->get( _T( "id" ) ) ).toInt();
+ QString result = QString::fromWCharArray( d->get( table.toStdWString().c_str() ) );
+
+ if ( result.toLower() == name.toLower() )
+ score = 1.0;
+ else
+ score = qMin( score, (float)0.99 );
+
+ if ( score > 0.05 )
+ {
+ resultsmap.insert( id, score );
+// qDebug() << "Hitres:" << result << id << score << table << name;
}
}
+
return resultsmap;
}
diff --git a/src/libtomahawk/database/fuzzyindex.h b/src/libtomahawk/database/fuzzyindex.h
index 0258493eb..df49ba31b 100644
--- a/src/libtomahawk/database/fuzzyindex.h
+++ b/src/libtomahawk/database/fuzzyindex.h
@@ -5,33 +5,59 @@
#include <QMap>
#include <QHash>
#include <QString>
+#include <QMutex>
+
+namespace lucene
+{
+ namespace analysis
+ {
+ class SimpleAnalyzer;
+ }
+ namespace store
+ {
+ class Directory;
+ }
+ namespace index
+ {
+ class IndexReader;
+ class IndexWriter;
+ }
+ namespace search
+ {
+ class IndexSearcher;
+ }
+}
class DatabaseImpl;
-
class FuzzyIndex : public QObject
{
Q_OBJECT
public:
- explicit FuzzyIndex( DatabaseImpl &db );
+ explicit FuzzyIndex( DatabaseImpl& db );
+ ~FuzzyIndex();
+ void beginIndexing();
+ void endIndexing();
+ void appendFields( const QString& table, const QMap< unsigned int, QString >& fields );
+
signals:
void indexReady();
public slots:
- void loadNgramIndex();
+ void loadLuceneIndex();
+
QMap< int, float > search( const QString& table, const QString& name );
- void mergeIndex( const QString& table, const QHash< QString, QMap<quint32, quint16> >& tomerge );
private:
- void loadNgramIndex_helper( QHash< QString, QMap<quint32, quint16> >& idx, const QString& table, unsigned int fromkey = 0 );
+ DatabaseImpl& m_db;
+ QMutex m_mutex;
- // maps an ngram to {track id, num occurences}
- QHash< QString, QMap<quint32, quint16> > m_artist_ngrams, m_album_ngrams, m_track_ngrams;
-
- DatabaseImpl & m_db;
- bool m_loaded;
+ lucene::analysis::SimpleAnalyzer* m_analyzer;
+ lucene::store::Directory* m_luceneDir;
+ lucene::index::IndexReader* m_luceneReader;
+ lucene::search::IndexSearcher* m_luceneSearcher;
};
#endif // FUZZYINDEX_H
diff --git a/src/libtomahawk/database/gen_schema.h.sh b/src/libtomahawk/database/gen_schema.h.sh
index b8a6a6b79..424ee18c0 100755
--- a/src/libtomahawk/database/gen_schema.h.sh
+++ b/src/libtomahawk/database/gen_schema.h.sh
@@ -1,4 +1,8 @@
#!/bin/bash
+
+# !!!! You need to manually generate schema.sql.h when the schema changes:
+# ./gen_schema.h.sh ./schema.sql tomahawk > schema.sql.h
+
schema=$1
name=$2
diff --git a/src/libtomahawk/database/schema.sql b/src/libtomahawk/database/schema.sql
index d6b586165..174634d6f 100644
--- a/src/libtomahawk/database/schema.sql
+++ b/src/libtomahawk/database/schema.sql
@@ -14,6 +14,8 @@ CREATE TABLE IF NOT EXISTS oplog (
CREATE UNIQUE INDEX oplog_guid ON oplog(guid);
CREATE INDEX oplog_source ON oplog(source);
+
+
-- the basic 3 catalogue tables:
CREATE TABLE IF NOT EXISTS artist (
@@ -39,6 +41,8 @@ CREATE TABLE IF NOT EXISTS album (
);
CREATE UNIQUE INDEX album_artist_sortname ON album(artist,sortname);
+
+
-- Source, typically a remote peer.
CREATE TABLE IF NOT EXISTS source (
@@ -51,6 +55,7 @@ CREATE TABLE IF NOT EXISTS source (
CREATE UNIQUE INDEX source_name ON source(name);
+
-- playlists
CREATE TABLE IF NOT EXISTS playlist (
@@ -83,16 +88,8 @@ CREATE TABLE IF NOT EXISTS playlist_item (
addedby INTEGER REFERENCES source(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED, -- who added this to the playlist
result_hint TEXT -- hint as to a result, to avoid using the resolver
);
-
CREATE INDEX playlist_item_playlist ON playlist_item(playlist);
---INSERT INTO playlist_item(guid, playlist, trackname, artistname)
--- VALUES('itemguid-1','playlistguid-1','track name 01','artist name 01');
---INSERT INTO playlist_item(guid, playlist, trackname, artistname)
--- VALUES('itemguid-2','playlistguid-1','track name 02','artist name 02');
---INSERT INTO playlist_item(guid, playlist, trackname, artistname)
--- VALUES('itemguid-3','playlistguid-1','track name 03','artist name 03');
---
CREATE TABLE IF NOT EXISTS playlist_revision (
guid TEXT PRIMARY KEY,
playlist TEXT NOT NULL REFERENCES playlist(guid) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
@@ -146,31 +143,6 @@ CREATE TABLE IF NOT EXISTS dynamic_playlist_revision (
--INSERT INTO dynamic_playlist_revision(guid, controls, plmode, pltype)
-- VALUES('revisionguid-11', '["controlid-2"]', 1, "echonest");
--- the trigram search indexes
-
-CREATE TABLE IF NOT EXISTS artist_search_index (
- ngram TEXT NOT NULL,
- id INTEGER NOT NULL REFERENCES artist(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
- num INTEGER NOT NULL DEFAULT 1,
- PRIMARY KEY(ngram, id)
-);
--- CREATE INDEX artist_search_index_ngram ON artist_search_index(ngram);
-
-CREATE TABLE IF NOT EXISTS album_search_index (
- ngram TEXT NOT NULL,
- id INTEGER NOT NULL REFERENCES album(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
- num INTEGER NOT NULL DEFAULT 1,
- PRIMARY KEY(ngram, id)
-);
--- CREATE INDEX album_search_index_ngram ON album_search_index(ngram);
-
-CREATE TABLE IF NOT EXISTS track_search_index (
- ngram TEXT NOT NULL,
- id INTEGER NOT NULL REFERENCES track(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
- num INTEGER NOT NULL DEFAULT 1,
- PRIMARY KEY(ngram, id)
-);
--- CREATE INDEX track_search_index_ngram ON track_search_index(ngram);
-- files on disk and joinage with catalogue. physical properties of files only:
@@ -196,7 +168,6 @@ CREATE TABLE IF NOT EXISTS dirs_scanned (
mtime INTEGER NOT NULL
);
-
CREATE TABLE IF NOT EXISTS file_join (
file INTEGER PRIMARY KEY REFERENCES file(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
artist INTEGER NOT NULL REFERENCES artist(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
@@ -211,9 +182,9 @@ CREATE INDEX file_join_album ON file_join(album);
-- tags, weighted and by source (rock, jazz etc)
+
-- weight is always 1.0 if tag provided by our user.
-- may be less from aggregate sources like lastfm global tags
-
CREATE TABLE IF NOT EXISTS track_tags (
id INTEGER PRIMARY KEY, -- track id
source INTEGER REFERENCES source(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
@@ -256,6 +227,7 @@ CREATE INDEX track_attrib_id ON track_attributes(id);
CREATE INDEX track_attrib_k ON track_attributes(k);
+
-- playback history
-- if source=null, file is local to this machine
@@ -270,6 +242,7 @@ CREATE INDEX playback_log_source ON playback_log(source);
CREATE INDEX playback_log_track ON playback_log(track);
+
-- Schema version, and misc tomahawk settings relating to the collection db
CREATE TABLE IF NOT EXISTS settings (
@@ -277,4 +250,4 @@ CREATE TABLE IF NOT EXISTS settings (
v TEXT NOT NULL DEFAULT ''
);
-INSERT INTO settings(k,v) VALUES('schema_version', '17');
+INSERT INTO settings(k,v) VALUES('schema_version', '19');
diff --git a/src/libtomahawk/database/schema.sql.h b/src/libtomahawk/database/schema.sql.h
index e1e9f32aa..d8052e10d 100644
--- a/src/libtomahawk/database/schema.sql.h
+++ b/src/libtomahawk/database/schema.sql.h
@@ -1,5 +1,5 @@
/*
- This file was automatically generated from schema.sql on Sun Jan 9 13:09:56 EST 2011.
+ This file was automatically generated from schema.sql on Thu Feb 3 08:21:21 EST 2011.
*/
static const char * tomahawk_schema_sql =
@@ -94,24 +94,6 @@ static const char * tomahawk_schema_sql =
" plmode INTEGER,"
" pltype TEXT"
");"
-"CREATE TABLE IF NOT EXISTS artist_search_index ("
-" ngram TEXT NOT NULL,"
-" id INTEGER NOT NULL REFERENCES artist(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,"
-" num INTEGER NOT NULL DEFAULT 1,"
-" PRIMARY KEY(ngram, id)"
-");"
-"CREATE TABLE IF NOT EXISTS album_search_index ("
-" ngram TEXT NOT NULL,"
-" id INTEGER NOT NULL REFERENCES album(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,"
-" num INTEGER NOT NULL DEFAULT 1,"
-" PRIMARY KEY(ngram, id)"
-");"
-"CREATE TABLE IF NOT EXISTS track_search_index ("
-" ngram TEXT NOT NULL,"
-" id INTEGER NOT NULL REFERENCES track(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,"
-" num INTEGER NOT NULL DEFAULT 1,"
-" PRIMARY KEY(ngram, id)"
-");"
"CREATE TABLE IF NOT EXISTS file ("
" id INTEGER PRIMARY KEY AUTOINCREMENT,"
" source INTEGER REFERENCES source(id) ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,"
@@ -183,7 +165,7 @@ static const char * tomahawk_schema_sql =
" k TEXT NOT NULL PRIMARY KEY,"
" v TEXT NOT NULL DEFAULT ''"
");"
-"INSERT INTO settings(k,v) VALUES('schema_version', '17');"
+"INSERT INTO settings(k,v) VALUES('schema_version', '19');"
;
const char * get_tomahawk_sql()
diff --git a/src/libtomahawk/pipeline.cpp b/src/libtomahawk/pipeline.cpp
index 158ceb203..d0c75db0b 100644
--- a/src/libtomahawk/pipeline.cpp
+++ b/src/libtomahawk/pipeline.cpp
@@ -29,7 +29,7 @@ Pipeline::Pipeline( QObject* parent )
void
Pipeline::databaseReady()
{
- connect( Database::instance(), SIGNAL(indexReady()), this, SLOT(indexReady()), Qt::QueuedConnection );
+ connect( Database::instance(), SIGNAL( indexReady() ), this, SLOT( indexReady() ), Qt::QueuedConnection );
Database::instance()->loadIndex();
}
@@ -73,7 +73,7 @@ Pipeline::addResolver( Resolver* r, bool sort )
void
-Pipeline::add( const QList<query_ptr>& qlist, bool prioritized )
+Pipeline::resolve( const QList<query_ptr>& qlist, bool prioritized )
{
{
QMutexLocker lock( &m_mut );
@@ -103,12 +103,21 @@ Pipeline::add( const QList<query_ptr>& qlist, bool prioritized )
void
-Pipeline::add( const query_ptr& q, bool prioritized )
+Pipeline::resolve( const query_ptr& q, bool prioritized )
{
- //qDebug() << Q_FUNC_INFO << (qlonglong)q.data() << q->toString();
+ if ( q.isNull() )
+ return;
+
QList< query_ptr > qlist;
qlist << q;
- add( qlist, prioritized );
+ resolve( qlist, prioritized );
+}
+
+
+void
+Pipeline::resolve( QID qid, bool prioritized )
+{
+ resolve( query( qid ), prioritized );
}
@@ -117,12 +126,22 @@ Pipeline::reportResults( QID qid, const QList< result_ptr >& results )
{
QMutexLocker lock( &m_mut );
- if( !m_qids.contains( qid ) )
+ if ( !m_qids.contains( qid ) )
{
qDebug() << "reportResults called for unknown QID";
return;
}
+ unsigned int state = m_qidsState.value( qid ) - 1;
+ m_qidsState.insert( qid, state );
+
+ if ( state == 0 )
+ {
+ // All resolvers have reported back their results for this query now
+ const query_ptr& q = m_qids.value( qid );
+ q->onResolvingFinished();
+ }
+
if ( !results.isEmpty() )
{
//qDebug() << Q_FUNC_INFO << qid;
@@ -189,6 +208,9 @@ Pipeline::shunt( const query_ptr& q )
// resolvers aren't allowed to block in this call:
//qDebug() << "Dispaching to resolver" << r->name();
+
+ unsigned int state = m_qidsState.value( q->id() );
+ m_qidsState.insert( q->id(), state + 1 );
r->resolve( q->toVariant() );
}
else
diff --git a/src/libtomahawk/pipeline.h b/src/libtomahawk/pipeline.h
index 873a91efa..615d6d1d5 100644
--- a/src/libtomahawk/pipeline.h
+++ b/src/libtomahawk/pipeline.h
@@ -27,9 +27,6 @@ public:
explicit Pipeline( QObject* parent = 0 );
-// const query_ptr& query( QID qid ) const;
-// result_ptr result( RID rid ) const;
-
void reportResults( QID qid, const QList< result_ptr >& results );
/// sorter to rank resolver priority
@@ -49,8 +46,9 @@ public:
}
public slots:
- void add( const query_ptr& q, bool prioritized = true );
- void add( const QList<query_ptr>& qlist, bool prioritized = true );
+ void resolve( const query_ptr& q, bool prioritized = true );
+ void resolve( const QList<query_ptr>& qlist, bool prioritized = true );
+ void resolve( QID qid, bool prioritized = true );
void databaseReady();
signals:
@@ -64,6 +62,8 @@ private slots:
private:
QList< Resolver* > m_resolvers;
+
+ QMap< QID, unsigned int > m_qidsState;
QMap< QID, query_ptr > m_qids;
QMap< RID, result_ptr > m_rids;
diff --git a/src/libtomahawk/playlist.cpp b/src/libtomahawk/playlist.cpp
index 614ae1e33..6f0d51400 100644
--- a/src/libtomahawk/playlist.cpp
+++ b/src/libtomahawk/playlist.cpp
@@ -22,7 +22,7 @@ PlaylistEntry::~PlaylistEntry() {}
void
PlaylistEntry::setQueryVariant( const QVariant& v )
{
- m_query = query_ptr( new Query( v ) );
+ m_query = Tomahawk::Query::get( v, false );
}
@@ -387,7 +387,8 @@ Playlist::resolve()
{
qlist << p->query();
}
- Pipeline::instance()->add( qlist );
+
+ Pipeline::instance()->resolve( qlist );
}
void
diff --git a/src/libtomahawk/playlist/collectionmodel.cpp b/src/libtomahawk/playlist/collectionmodel.cpp
index d951b4aed..212f60c79 100644
--- a/src/libtomahawk/playlist/collectionmodel.cpp
+++ b/src/libtomahawk/playlist/collectionmodel.cpp
@@ -195,7 +195,7 @@ CollectionModel::onTracksAdded( const QList<QVariant>& tracks, const collection_
PlItem* plitem;
foreach( const QVariant& v, tracks )
{
- Tomahawk::query_ptr query = query_ptr( new Query( v ) );
+ Tomahawk::query_ptr query = Tomahawk::Query::get( v, false );
// FIXME: needs merging
// Manually add a result, since it's coming from the local collection
diff --git a/src/libtomahawk/playlist/dynamic/echonest/EchonestGenerator.cpp b/src/libtomahawk/playlist/dynamic/echonest/EchonestGenerator.cpp
index e334fdaab..6c7176710 100644
--- a/src/libtomahawk/playlist/dynamic/echonest/EchonestGenerator.cpp
+++ b/src/libtomahawk/playlist/dynamic/echonest/EchonestGenerator.cpp
@@ -276,7 +276,7 @@ EchonestGenerator::queryFromSong(const Echonest::Song& song)
track[ "artist" ] = song.artistName();
// track[ "album" ] = song.release(); // TODO should we include it? can be quite specific
track[ "track" ] = song.title();
- return query_ptr( new Query( track ) );
+ return Query::get( track );
}
QWidget*
diff --git a/src/libtomahawk/playlist/dynamic/widgets/DynamicWidget.cpp b/src/libtomahawk/playlist/dynamic/widgets/DynamicWidget.cpp
index 30c885389..af3f7971c 100644
--- a/src/libtomahawk/playlist/dynamic/widgets/DynamicWidget.cpp
+++ b/src/libtomahawk/playlist/dynamic/widgets/DynamicWidget.cpp
@@ -268,7 +268,6 @@ DynamicWidget::tracksGenerated( const QList< query_ptr >& queries )
} else { // read-only, so add tracks only in the GUI, not to the playlist itself
foreach( const query_ptr& query, queries ) {
m_model->append( query );
- Pipeline::instance()->add( query );
}
}
}
@@ -280,7 +279,6 @@ DynamicWidget::onDemandFetched( const Tomahawk::query_ptr& track )
connect( track.data(), SIGNAL( resultsAdded( QList<Tomahawk::result_ptr> ) ), this, SLOT( trackResolved() ) );
m_model->append( track );
- Pipeline::instance()->add( track );
}
void
diff --git a/src/libtomahawk/playlist/topbar/topbar.ui b/src/libtomahawk/playlist/topbar/topbar.ui
index 6d5ee8c7f..b817c341c 100644
--- a/src/libtomahawk/playlist/topbar/topbar.ui
+++ b/src/libtomahawk/playlist/topbar/topbar.ui
@@ -316,7 +316,7 @@
<customwidget>
<class>SearchLineEdit</class>
<extends>QLineEdit</extends>
- <header>topbar/searchlineedit.h</header>
+ <header>playlist/topbar/searchlineedit.h</header>
</customwidget>
</customwidgets>
<resources/>
diff --git a/src/libtomahawk/playlistinterface.h b/src/libtomahawk/playlistinterface.h
index 64c871feb..c9b375f07 100644
--- a/src/libtomahawk/playlistinterface.h
+++ b/src/libtomahawk/playlistinterface.h
@@ -7,6 +7,7 @@
#include "typedefs.h"
#include "dllmacro.h"
+#include "result.h"
class DLLEXPORT PlaylistInterface
{
diff --git a/src/libtomahawk/query.cpp b/src/libtomahawk/query.cpp
index 420cd049b..39c26fa9f 100644
--- a/src/libtomahawk/query.cpp
+++ b/src/libtomahawk/query.cpp
@@ -3,9 +3,24 @@
#include "collection.h"
#include <QtAlgorithms>
+#include "database/database.h"
+#include "pipeline.h"
+#include "sourcelist.h"
+
using namespace Tomahawk;
+query_ptr
+Query::get( const QVariant& v, bool autoResolve )
+{
+ query_ptr q = query_ptr( new Query( v ) );
+
+ if ( autoResolve )
+ Pipeline::instance()->resolve( q );
+ return q;
+}
+
+
Query::Query( const QVariant& v )
: m_v( v )
, m_solved( false )
@@ -17,6 +32,9 @@ Query::Query( const QVariant& v )
m_track = m.value( "track" ).toString();
m_qid = m.value( "qid" ).toString();
+
+ connect( SourceList::instance(), SIGNAL( sourceAdded( Tomahawk::source_ptr ) ), SLOT( refreshResults() ), Qt::QueuedConnection );
+ connect( Database::instance(), SIGNAL( indexReady() ), SLOT( refreshResults() ), Qt::QueuedConnection );
}
@@ -46,6 +64,13 @@ Query::addResults( const QList< Tomahawk::result_ptr >& newresults )
}
+void
+Query::refreshResults()
+{
+ Pipeline::instance()->resolve( id() );
+}
+
+
void
Query::resultUnavailable()
{
@@ -83,6 +108,14 @@ Query::removeResult( const Tomahawk::result_ptr& result )
}
+void
+Query::onResolvingFinished()
+{
+// qDebug() << Q_FUNC_INFO << "Finished resolving." << toString();
+ emit resolvingFinished( !m_results.isEmpty() );
+}
+
+
QList< result_ptr >
Query::results() const
{
diff --git a/src/libtomahawk/query.h b/src/libtomahawk/query.h
index e5ac29b3d..83a4a74e6 100644
--- a/src/libtomahawk/query.h
+++ b/src/libtomahawk/query.h
@@ -19,6 +19,7 @@ class DLLEXPORT Query : public QObject
Q_OBJECT
public:
+ static query_ptr get( const QVariant& v, bool autoResolve = true );
explicit Query( const QVariant& v );
QVariant toVariant() const { return m_v; }
@@ -38,7 +39,7 @@ public:
bool solved() const { return m_solved; }
unsigned int lastPipelineWeight() const { return m_lastpipelineweight; }
- void setLastPipelineWeight( unsigned int w ) { m_lastpipelineweight = w;}
+ void setLastPipelineWeight( unsigned int w ) { m_lastpipelineweight = w; }
/// for debug output:
QString toString() const
@@ -53,16 +54,20 @@ public:
signals:
void resultsAdded( const QList<Tomahawk::result_ptr>& );
void resultsRemoved( const Tomahawk::result_ptr& );
- void solvedStateChanged( bool state );
- void resolveFailed();
+ void solvedStateChanged( bool state );
+ void resolvingFinished( bool hasResults );
+
public slots:
/// (indirectly) called by resolver plugins when results are found
void addResults( const QList< Tomahawk::result_ptr >& );
void removeResult( const Tomahawk::result_ptr& );
+ void onResolvingFinished();
+
private slots:
void resultUnavailable();
+ void refreshResults();
private:
mutable QMutex m_mut;
diff --git a/src/libtomahawk/result.cpp b/src/libtomahawk/result.cpp
index 88146df4f..fa129215d 100644
--- a/src/libtomahawk/result.cpp
+++ b/src/libtomahawk/result.cpp
@@ -80,7 +80,7 @@ Result::toString() const
Tomahawk::query_ptr
Result::toQuery() const
{
- Tomahawk::query_ptr query = Tomahawk::query_ptr( new Tomahawk::Query( toVariant() ) );
+ Tomahawk::query_ptr query = Tomahawk::Query::get( toVariant(), false );
return query;
}
diff --git a/src/libtomahawk/utils/xspfloader.cpp b/src/libtomahawk/utils/xspfloader.cpp
index 2397ee648..f8125c945 100644
--- a/src/libtomahawk/utils/xspfloader.cpp
+++ b/src/libtomahawk/utils/xspfloader.cpp
@@ -105,7 +105,7 @@ XSPFLoader::gotBody()
v.insert( "album", e.firstChildElement( "album" ).text() );
v.insert( "track", e.firstChildElement( "title" ).text() );
- p->setQuery( Tomahawk::query_ptr(new Tomahawk::Query(v)) );
+ p->setQuery( Tomahawk::Query::get( v ) );
m_entries << p;
}
diff --git a/src/libtomahawk/widgets/newplaylistwidget.cpp b/src/libtomahawk/widgets/newplaylistwidget.cpp
index 75c5d065f..7882532c0 100644
--- a/src/libtomahawk/widgets/newplaylistwidget.cpp
+++ b/src/libtomahawk/widgets/newplaylistwidget.cpp
@@ -11,7 +11,6 @@
#include "widgets/overlaywidget.h"
-#include "pipeline.h"
#include "utils/xspfloader.h"
#include "sourcelist.h"
@@ -115,8 +114,6 @@ NewPlaylistWidget::suggestionsFound()
ql.append( entry->query() );
}
- Tomahawk::Pipeline::instance()->add( ql );
-
loader->deleteLater();
}
diff --git a/src/tomahawkapp.cpp b/src/tomahawkapp.cpp
index ef5af8c2a..eadf33f5a 100644
--- a/src/tomahawkapp.cpp
+++ b/src/tomahawkapp.cpp
@@ -320,8 +320,7 @@ void
TomahawkApp::setupPipeline()
{
// setup resolvers for local content, and (cached) remote collection content
- Pipeline::instance()->addResolver( new DatabaseResolver( true, 100 ) );
- Pipeline::instance()->addResolver( new DatabaseResolver( false, 90 ) );
+ Pipeline::instance()->addResolver( new DatabaseResolver( 100 ) );
// new ScriptResolver("/home/rj/src/tomahawk-core/contrib/magnatune/magnatune-resolver.php");
}
diff --git a/src/tomahawkwindow.cpp b/src/tomahawkwindow.cpp
index ac0296653..784dd3571 100644
--- a/src/tomahawkwindow.cpp
+++ b/src/tomahawkwindow.cpp
@@ -390,5 +390,5 @@ TomahawkWindow::setWindowTitle( const QString& title )
void
TomahawkWindow::showAboutTomahawk()
{
- QMessageBox::about( this, "About Tomahawk", "Copyright 2010 Christian Muehlhaeuser <muesli@gmail.com>\n\nThanks to: Leo Franchi, Jeff Mitchell, Dominik Schmidt, Alejandro Wainzinger and Steve Robertson" );
+ QMessageBox::about( this, "About Tomahawk", "Copyright 2010, 2011 Christian Muehlhaeuser <muesli@gmail.com>\n\nThanks to: Leo Franchi, Jeff Mitchell, Dominik Schmidt, Alejandro Wainzinger and Steve Robertson" );
}
diff --git a/src/web/api_v1.h b/src/web/api_v1.h
index 0a77a679d..6d89beddd 100644
--- a/src/web/api_v1.h
+++ b/src/web/api_v1.h
@@ -112,8 +112,7 @@ public slots:
m.insert( "track", event->url.queryItemValue("track") );
m.insert( "qid", qid );
- Tomahawk::query_ptr qry( new Tomahawk::Query( m ) );
- Tomahawk::Pipeline::instance()->add( qry );
+ Tomahawk::query_ptr qry = Tomahawk::Query::get( m );
QVariantMap r;
r.insert( "qid", qid );
diff --git a/src/xmppbot/xmppbot.cpp b/src/xmppbot/xmppbot.cpp
index 0745a07e8..ccb7cc40c 100644
--- a/src/xmppbot/xmppbot.cpp
+++ b/src/xmppbot/xmppbot.cpp
@@ -5,7 +5,6 @@
#include "album.h"
#include "typedefs.h"
#include "tomahawksettings.h"
-#include "pipeline.h"
#include "audio/audioengine.h"
@@ -145,11 +144,10 @@ void XMPPBot::handleMessage(const Message& msg, MessageSession* session)
QVariantMap qv;
qv["artist"] = tokens.first().trimmed();
qv["track"] = tokens.last().trimmed();
- Tomahawk::query_ptr q( new Tomahawk::Query( qv ) );
+ Tomahawk::query_ptr q = Tomahawk::Query::get( qv );
connect( q.data(), SIGNAL( resultsAdded( QList<Tomahawk::result_ptr> ) ),
SLOT( onResultsAdded( QList<Tomahawk::result_ptr> ) ) );
- Tomahawk::Pipeline::instance()->add( q );
return;
}
else if ( body.startsWith( "stop" ) )