From af6f96728f20919d54fc160a699363bc15558e51 Mon Sep 17 00:00:00 2001 From: Christian Muehlhaeuser Date: Tue, 7 May 2013 04:23:46 +0200 Subject: [PATCH] * Worked around Lucene memory leak and improved our own memory foot-print during indexing. --- .../DatabaseCommand_UpdateSearchIndex.cpp | 31 ++++--- .../DatabaseCommand_UpdateSearchIndex.h | 11 ++- src/libtomahawk/database/FuzzyIndex.cpp | 82 ++++++++----------- src/libtomahawk/database/FuzzyIndex.h | 8 +- 4 files changed, 65 insertions(+), 67 deletions(-) diff --git a/src/libtomahawk/database/DatabaseCommand_UpdateSearchIndex.cpp b/src/libtomahawk/database/DatabaseCommand_UpdateSearchIndex.cpp index 40e14c781..3f0b86df4 100644 --- a/src/libtomahawk/database/DatabaseCommand_UpdateSearchIndex.cpp +++ b/src/libtomahawk/database/DatabaseCommand_UpdateSearchIndex.cpp @@ -1,6 +1,6 @@ /* === This file is part of Tomahawk Player - === * - * Copyright 2010-2011, Christian Muehlhaeuser + * Copyright 2010-2013, Christian Muehlhaeuser * Copyright 2012 Leo Franchi * * Tomahawk is free software: you can redistribute it and/or modify @@ -49,6 +49,8 @@ DatabaseCommand_UpdateSearchIndex::DatabaseCommand_UpdateSearchIndex() DatabaseCommand_UpdateSearchIndex::~DatabaseCommand_UpdateSearchIndex() { + tDebug() << Q_FUNC_INFO; + #ifndef ENABLE_HEADLESS if ( ! m_statusJob.isNull() ) m_statusJob.data()->done(); @@ -61,35 +63,30 @@ DatabaseCommand_UpdateSearchIndex::exec( DatabaseImpl* db ) { db->m_fuzzyIndex->beginIndexing(); - QMap< unsigned int, QMap< QString, QString > > data; TomahawkSqlQuery q = db->newquery(); - q.exec( "SELECT track.id, track.name, artist.name, artist.id FROM track, artist WHERE artist.id = track.artist" ); while ( q.next() ) { - QMap< QString, QString > track; - track.insert( "track", q.value( 1 ).toString() ); - track.insert( "artist", q.value( 2 ).toString() ); - track.insert( "artistid", q.value( 3 ).toString() ); + IndexData ida; + ida.id = q.value( 0 ).toUInt(); + ida.artistId = q.value( 3 ).toUInt(); + ida.track = q.value( 1 ).toString(); + ida.artist = q.value( 2 ).toString(); - data.insert( q.value( 0 ).toUInt(), track ); + db->m_fuzzyIndex->appendFields( ida ); } - db->m_fuzzyIndex->appendFields( data ); - data.clear(); - q.exec( "SELECT album.id, album.name FROM album" ); while ( q.next() ) { - QMap< QString, QString > album; - album.insert( "album", q.value( 1 ).toString() ); + IndexData ida; + ida.id = q.value( 0 ).toUInt(); + ida.album = q.value( 1 ).toString(); - data.insert( q.value( 0 ).toUInt(), album ); + db->m_fuzzyIndex->appendFields( ida ); } - db->m_fuzzyIndex->appendFields( data ); - - qDebug() << "Building index finished."; + tDebug( LOGVERBOSE ) << "Building index finished."; db->m_fuzzyIndex->endIndexing(); } diff --git a/src/libtomahawk/database/DatabaseCommand_UpdateSearchIndex.h b/src/libtomahawk/database/DatabaseCommand_UpdateSearchIndex.h index 90537b998..6819c9529 100644 --- a/src/libtomahawk/database/DatabaseCommand_UpdateSearchIndex.h +++ b/src/libtomahawk/database/DatabaseCommand_UpdateSearchIndex.h @@ -1,6 +1,6 @@ /* === This file is part of Tomahawk Player - === * - * Copyright 2010-2011, Christian Muehlhaeuser + * Copyright 2010-2013, Christian Muehlhaeuser * * Tomahawk is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -25,6 +25,15 @@ class IndexingJobItem; +struct IndexData +{ + unsigned int id; + unsigned int artistId; + QString artist; + QString album; + QString track; +}; + class DLLEXPORT DatabaseCommand_UpdateSearchIndex : public DatabaseCommand { Q_OBJECT diff --git a/src/libtomahawk/database/FuzzyIndex.cpp b/src/libtomahawk/database/FuzzyIndex.cpp index 176e55ccc..19826dc7a 100644 --- a/src/libtomahawk/database/FuzzyIndex.cpp +++ b/src/libtomahawk/database/FuzzyIndex.cpp @@ -1,6 +1,6 @@ /* === This file is part of Tomahawk Player - === * - * Copyright 2010-2011, Christian Muehlhaeuser + * Copyright 2010-2013, Christian Muehlhaeuser * * Tomahawk is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -24,7 +24,6 @@ #include #include -#include "DatabaseCommand_UpdateSearchIndex.h" #include "DatabaseImpl.h" #include "Database.h" #include "utils/TomahawkUtils.h" @@ -112,10 +111,11 @@ FuzzyIndex::beginIndexing() try { - qDebug() << Q_FUNC_INFO << "Starting indexing."; + tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Starting indexing."; if ( m_luceneReader != 0 ) { - qDebug() << "Deleting old lucene stuff."; + tDebug( LOGVERBOSE ) << "Deleting old lucene stuff."; + m_luceneSearcher->close(); m_luceneReader->close(); delete m_luceneSearcher; @@ -124,8 +124,8 @@ FuzzyIndex::beginIndexing() m_luceneReader = 0; } - qDebug() << "Creating new index writer."; - IndexWriter luceneWriter( m_luceneDir, m_analyzer, true ); + tDebug( LOGVERBOSE ) << "Creating new index writer."; + m_luceneWriter = new IndexWriter( m_luceneDir, m_analyzer, true ); } catch( CLuceneError& error ) { @@ -138,62 +138,52 @@ FuzzyIndex::beginIndexing() void FuzzyIndex::endIndexing() { + m_luceneWriter->optimize(); + m_luceneWriter->close(); + delete m_luceneWriter; + m_luceneWriter = 0; + m_mutex.unlock(); emit indexReady(); } void -FuzzyIndex::appendFields( const QMap< unsigned int, QMap< QString, QString > >& trackData ) +FuzzyIndex::appendFields( const IndexData& data ) { try { - tDebug() << "Appending to index:" << trackData.count(); - bool create = !IndexReader::indexExists( TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" ).toStdString().c_str() ); - IndexWriter luceneWriter( m_luceneDir, m_analyzer, create ); Document doc; - QMapIterator< unsigned int, QMap< QString, QString > > it( trackData ); - while ( it.hasNext() ) + if ( !data.track.isEmpty() ) { - it.next(); - unsigned int id = it.key(); - QMap< QString, QString > values = it.value(); + doc.add( *( _CLNEW Field( _T( "fulltext" ), DatabaseImpl::sortname( QString( "%1 %2" ).arg( data.artist ).arg( data.track ) ).toStdWString().c_str(), + Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) ); - if ( values.contains( "track" ) ) - { - doc.add( *( _CLNEW Field( _T( "fulltext" ), DatabaseImpl::sortname( QString( "%1 %2" ).arg( values.value( "artist" ) ).arg( values.value( "track" ) ) ).toStdWString().c_str(), - Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) ); + doc.add( *( _CLNEW Field( _T( "track" ), DatabaseImpl::sortname( data.track ).toStdWString().c_str(), + Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) ); - doc.add( *( _CLNEW Field( _T( "track" ), DatabaseImpl::sortname( values.value( "track" ) ).toStdWString().c_str(), - Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) ); + doc.add( *( _CLNEW Field( _T( "artist" ), DatabaseImpl::sortname( data.artist ).toStdWString().c_str(), + Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) ); - doc.add( *( _CLNEW Field( _T( "artist" ), DatabaseImpl::sortname( values.value( "artist" ) ).toStdWString().c_str(), - Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) ); + doc.add( *( _CLNEW Field( _T( "artistid" ), QString::number( data.artistId ).toStdWString().c_str(), + Field::STORE_YES | Field::INDEX_NO ) ) ); - doc.add( *( _CLNEW Field( _T( "artistid" ), values.value( "artistid" ).toStdWString().c_str(), - Field::STORE_YES | Field::INDEX_NO ) ) ); - - doc.add( *( _CLNEW Field( _T( "trackid" ), QString::number( id ).toStdWString().c_str(), - Field::STORE_YES | Field::INDEX_NO ) ) ); - } - else if ( values.contains( "album" ) ) - { - doc.add( *( _CLNEW Field( _T( "album" ), DatabaseImpl::sortname( values.value( "album" ) ).toStdWString().c_str(), - Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) ); - - doc.add( *( _CLNEW Field( _T( "albumid" ), QString::number( id ).toStdWString().c_str(), - Field::STORE_YES | Field::INDEX_NO ) ) ); - } - else - Q_ASSERT( false ); - - luceneWriter.addDocument( &doc ); - doc.clear(); + doc.add( *( _CLNEW Field( _T( "trackid" ), QString::number( data.id ).toStdWString().c_str(), + Field::STORE_YES | Field::INDEX_NO ) ) ); } + else if ( !data.album.isEmpty() ) + { + doc.add( *( _CLNEW Field( _T( "album" ), DatabaseImpl::sortname( data.album ).toStdWString().c_str(), + Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) ); - luceneWriter.optimize(); - luceneWriter.close(); + doc.add( *( _CLNEW Field( _T( "albumid" ), QString::number( data.id ).toStdWString().c_str(), + Field::STORE_YES | Field::INDEX_NO ) ) ); + } + else + Q_ASSERT( false ); + + m_luceneWriter->addDocument( &doc ); } catch( CLuceneError& error ) { @@ -223,7 +213,7 @@ FuzzyIndex::search( const Tomahawk::query_ptr& query ) { if ( !IndexReader::indexExists( TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" ).toStdString().c_str() ) ) { - qDebug() << Q_FUNC_INFO << "index didn't exist."; + tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "index didn't exist."; return resultsmap; } @@ -314,7 +304,7 @@ FuzzyIndex::searchAlbum( const Tomahawk::query_ptr& query ) { if ( !IndexReader::indexExists( TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" ).toStdString().c_str() ) ) { - qDebug() << Q_FUNC_INFO << "index didn't exist."; + tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "index didn't exist."; return resultsmap; } diff --git a/src/libtomahawk/database/FuzzyIndex.h b/src/libtomahawk/database/FuzzyIndex.h index 96af77cf8..cc5e00067 100644 --- a/src/libtomahawk/database/FuzzyIndex.h +++ b/src/libtomahawk/database/FuzzyIndex.h @@ -1,6 +1,6 @@ /* === This file is part of Tomahawk Player - === - * - * Copyright 2010-2011, Christian Muehlhaeuser + * + * Copyright 2010-2013, Christian Muehlhaeuser * * Tomahawk is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -26,6 +26,7 @@ #include #include "Query.h" +#include "DatabaseCommand_UpdateSearchIndex.h" namespace lucene { @@ -60,7 +61,7 @@ public: void beginIndexing(); void endIndexing(); - void appendFields( const QMap< unsigned int, QMap< QString, QString > >& trackData ); + void appendFields( const IndexData& data ); signals: void indexReady(); @@ -82,6 +83,7 @@ private: lucene::analysis::SimpleAnalyzer* m_analyzer; lucene::store::Directory* m_luceneDir; lucene::index::IndexReader* m_luceneReader; + lucene::index::IndexWriter* m_luceneWriter; lucene::search::IndexSearcher* m_luceneSearcher; };