mirror/tomahawk
1
0
Fork 0
mirror of https://github.com/tomahawk-player/tomahawk.git synced 2025-08-11 16:44:05 +02:00
Code Issues Releases Wiki Activity

* Worked around Lucene memory leak and improved our own memory foot-print during indexing.

Browse Source
This commit is contained in:
Christian Muehlhaeuser
2013-05-07 04:23:46 +02:00
parent c04fde0727
commit 64cf40ce8f
4 changed files with 65 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
src/libtomahawk/database/DatabaseCommand_UpdateSearchIndex.cpp
View File

@@ -1,6 +1,6 @@
/* === This file is part of Tomahawk Player - <http://tomahawk-player.org> === /* === This file is part of Tomahawk Player - <http://tomahawk-player.org> ===
* *
* Copyright 2010-2011, Christian Muehlhaeuser <muesli@tomahawk-player.org> * Copyright 2010-2013, Christian Muehlhaeuser <muesli@tomahawk-player.org>
* Copyright 2012 Leo Franchi <lfranchi@kde.org> * Copyright 2012 Leo Franchi <lfranchi@kde.org>
* *
* Tomahawk is free software: you can redistribute it and/or modify * Tomahawk is free software: you can redistribute it and/or modify
@@ -49,6 +49,8 @@ DatabaseCommand_UpdateSearchIndex::DatabaseCommand_UpdateSearchIndex()
DatabaseCommand_UpdateSearchIndex::~DatabaseCommand_UpdateSearchIndex() DatabaseCommand_UpdateSearchIndex::~DatabaseCommand_UpdateSearchIndex()
{ {
tDebug() << Q_FUNC_INFO;
#ifndef ENABLE_HEADLESS #ifndef ENABLE_HEADLESS
if ( ! m_statusJob.isNull() ) if ( ! m_statusJob.isNull() )
m_statusJob.data()->done(); m_statusJob.data()->done();
@@ -61,35 +63,30 @@ DatabaseCommand_UpdateSearchIndex::exec( DatabaseImpl* db )
{ {
db->m_fuzzyIndex->beginIndexing(); db->m_fuzzyIndex->beginIndexing();
QMap< unsigned int, QMap< QString, QString > > data;
TomahawkSqlQuery q = db->newquery(); TomahawkSqlQuery q = db->newquery();
q.exec( "SELECT track.id, track.name, artist.name, artist.id FROM track, artist WHERE artist.id = track.artist" ); q.exec( "SELECT track.id, track.name, artist.name, artist.id FROM track, artist WHERE artist.id = track.artist" );
while ( q.next() ) while ( q.next() )
{ {
QMap< QString, QString > track; IndexData ida;
track.insert( "track", q.value( 1 ).toString() ); ida.id = q.value( 0 ).toUInt();
track.insert( "artist", q.value( 2 ).toString() ); ida.artistId = q.value( 3 ).toUInt();
track.insert( "artistid", q.value( 3 ).toString() ); ida.track = q.value( 1 ).toString();
ida.artist = q.value( 2 ).toString();
data.insert( q.value( 0 ).toUInt(), track ); db->m_fuzzyIndex->appendFields( ida );
} }
db->m_fuzzyIndex->appendFields( data );
data.clear();
q.exec( "SELECT album.id, album.name FROM album" ); q.exec( "SELECT album.id, album.name FROM album" );
while ( q.next() ) while ( q.next() )
{ {
QMap< QString, QString > album; IndexData ida;
album.insert( "album", q.value( 1 ).toString() ); ida.id = q.value( 0 ).toUInt();
ida.album = q.value( 1 ).toString();
data.insert( q.value( 0 ).toUInt(), album ); db->m_fuzzyIndex->appendFields( ida );
} }
db->m_fuzzyIndex->appendFields( data ); tDebug( LOGVERBOSE ) << "Building index finished.";
qDebug() << "Building index finished.";
db->m_fuzzyIndex->endIndexing(); db->m_fuzzyIndex->endIndexing();
} }

11
src/libtomahawk/database/DatabaseCommand_UpdateSearchIndex.h
View File

@@ -1,6 +1,6 @@
/* === This file is part of Tomahawk Player - <http://tomahawk-player.org> === /* === This file is part of Tomahawk Player - <http://tomahawk-player.org> ===
* *
* Copyright 2010-2011, Christian Muehlhaeuser <muesli@tomahawk-player.org> * Copyright 2010-2013, Christian Muehlhaeuser <muesli@tomahawk-player.org>
* *
* Tomahawk is free software: you can redistribute it and/or modify * Tomahawk is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@@ -25,6 +25,15 @@
class IndexingJobItem; class IndexingJobItem;
struct IndexData
{
unsigned int id;
unsigned int artistId;
QString artist;
QString album;
QString track;
};
class DLLEXPORT DatabaseCommand_UpdateSearchIndex : public DatabaseCommand class DLLEXPORT DatabaseCommand_UpdateSearchIndex : public DatabaseCommand
{ {
Q_OBJECT Q_OBJECT

58
src/libtomahawk/database/FuzzyIndex.cpp
View File

@@ -1,6 +1,6 @@
/* === This file is part of Tomahawk Player - <http://tomahawk-player.org> === /* === This file is part of Tomahawk Player - <http://tomahawk-player.org> ===
* *
* Copyright 2010-2011, Christian Muehlhaeuser <muesli@tomahawk-player.org> * Copyright 2010-2013, Christian Muehlhaeuser <muesli@tomahawk-player.org>
* *
* Tomahawk is free software: you can redistribute it and/or modify * Tomahawk is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@@ -24,7 +24,6 @@
#include <CLucene.h> #include <CLucene.h>
#include <CLucene/queryParser/MultiFieldQueryParser.h> #include <CLucene/queryParser/MultiFieldQueryParser.h>
#include "DatabaseCommand_UpdateSearchIndex.h"
#include "DatabaseImpl.h" #include "DatabaseImpl.h"
#include "Database.h" #include "Database.h"
#include "utils/TomahawkUtils.h" #include "utils/TomahawkUtils.h"
@@ -112,10 +111,11 @@ FuzzyIndex::beginIndexing()
try try
{ {
qDebug() << Q_FUNC_INFO << "Starting indexing."; tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Starting indexing.";
if ( m_luceneReader != 0 ) if ( m_luceneReader != 0 )
{ {
qDebug() << "Deleting old lucene stuff."; tDebug( LOGVERBOSE ) << "Deleting old lucene stuff.";
m_luceneSearcher->close(); m_luceneSearcher->close();
m_luceneReader->close(); m_luceneReader->close();
delete m_luceneSearcher; delete m_luceneSearcher;
@@ -124,8 +124,8 @@ FuzzyIndex::beginIndexing()
m_luceneReader = 0; m_luceneReader = 0;
} }
qDebug() << "Creating new index writer."; tDebug( LOGVERBOSE ) << "Creating new index writer.";
IndexWriter luceneWriter( m_luceneDir, m_analyzer, true ); m_luceneWriter = new IndexWriter( m_luceneDir, m_analyzer, true );
} }
catch( CLuceneError& error ) catch( CLuceneError& error )
{ {
@@ -138,62 +138,52 @@ FuzzyIndex::beginIndexing()
void void
FuzzyIndex::endIndexing() FuzzyIndex::endIndexing()
{ {
m_luceneWriter->optimize();
m_luceneWriter->close();
delete m_luceneWriter;
m_luceneWriter = 0;
m_mutex.unlock(); m_mutex.unlock();
emit indexReady(); emit indexReady();
} }
void void
FuzzyIndex::appendFields( const QMap< unsigned int, QMap< QString, QString > >& trackData ) FuzzyIndex::appendFields( const IndexData& data )
{ {
try try
{ {
tDebug() << "Appending to index:" << trackData.count();
bool create = !IndexReader::indexExists( TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" ).toStdString().c_str() );
IndexWriter luceneWriter( m_luceneDir, m_analyzer, create );
Document doc; Document doc;
QMapIterator< unsigned int, QMap< QString, QString > > it( trackData ); if ( !data.track.isEmpty() )
while ( it.hasNext() )
{ {
it.next(); doc.add( *( _CLNEW Field( _T( "fulltext" ), DatabaseImpl::sortname( QString( "%1 %2" ).arg( data.artist ).arg( data.track ) ).toStdWString().c_str(),
unsigned int id = it.key();
QMap< QString, QString > values = it.value();
if ( values.contains( "track" ) )
{
doc.add( *( _CLNEW Field( _T( "fulltext" ), DatabaseImpl::sortname( QString( "%1 %2" ).arg( values.value( "artist" ) ).arg( values.value( "track" ) ) ).toStdWString().c_str(),
Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) ); Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) );
doc.add( *( _CLNEW Field( _T( "track" ), DatabaseImpl::sortname( values.value( "track" ) ).toStdWString().c_str(), doc.add( *( _CLNEW Field( _T( "track" ), DatabaseImpl::sortname( data.track ).toStdWString().c_str(),
Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) ); Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) );
doc.add( *( _CLNEW Field( _T( "artist" ), DatabaseImpl::sortname( values.value( "artist" ) ).toStdWString().c_str(), doc.add( *( _CLNEW Field( _T( "artist" ), DatabaseImpl::sortname( data.artist ).toStdWString().c_str(),
Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) ); Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) );
doc.add( *( _CLNEW Field( _T( "artistid" ), values.value( "artistid" ).toStdWString().c_str(), doc.add( *( _CLNEW Field( _T( "artistid" ), QString::number( data.artistId ).toStdWString().c_str(),
Field::STORE_YES | Field::INDEX_NO ) ) ); Field::STORE_YES | Field::INDEX_NO ) ) );
doc.add( *( _CLNEW Field( _T( "trackid" ), QString::number( id ).toStdWString().c_str(), doc.add( *( _CLNEW Field( _T( "trackid" ), QString::number( data.id ).toStdWString().c_str(),
Field::STORE_YES | Field::INDEX_NO ) ) ); Field::STORE_YES | Field::INDEX_NO ) ) );
} }
else if ( values.contains( "album" ) ) else if ( !data.album.isEmpty() )
{ {
doc.add( *( _CLNEW Field( _T( "album" ), DatabaseImpl::sortname( values.value( "album" ) ).toStdWString().c_str(), doc.add( *( _CLNEW Field( _T( "album" ), DatabaseImpl::sortname( data.album ).toStdWString().c_str(),
Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) ); Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) );
doc.add( *( _CLNEW Field( _T( "albumid" ), QString::number( id ).toStdWString().c_str(), doc.add( *( _CLNEW Field( _T( "albumid" ), QString::number( data.id ).toStdWString().c_str(),
Field::STORE_YES | Field::INDEX_NO ) ) ); Field::STORE_YES | Field::INDEX_NO ) ) );
} }
else else
Q_ASSERT( false ); Q_ASSERT( false );
luceneWriter.addDocument( &doc ); m_luceneWriter->addDocument( &doc );
doc.clear();
}
luceneWriter.optimize();
luceneWriter.close();
} }
catch( CLuceneError& error ) catch( CLuceneError& error )
{ {
@@ -223,7 +213,7 @@ FuzzyIndex::search( const Tomahawk::query_ptr& query )
{ {
if ( !IndexReader::indexExists( TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" ).toStdString().c_str() ) ) if ( !IndexReader::indexExists( TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" ).toStdString().c_str() ) )
{ {
qDebug() << Q_FUNC_INFO << "index didn't exist."; tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "index didn't exist.";
return resultsmap; return resultsmap;
} }
@@ -314,7 +304,7 @@ FuzzyIndex::searchAlbum( const Tomahawk::query_ptr& query )
{ {
if ( !IndexReader::indexExists( TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" ).toStdString().c_str() ) ) if ( !IndexReader::indexExists( TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" ).toStdString().c_str() ) )
{ {
qDebug() << Q_FUNC_INFO << "index didn't exist."; tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "index didn't exist.";
return resultsmap; return resultsmap;
} }

6
src/libtomahawk/database/FuzzyIndex.h
View File

@@ -1,6 +1,6 @@
/* === This file is part of Tomahawk Player - <http://tomahawk-player.org> === /* === This file is part of Tomahawk Player - <http://tomahawk-player.org> ===
* *
* Copyright 2010-2011, Christian Muehlhaeuser <muesli@tomahawk-player.org> * Copyright 2010-2013, Christian Muehlhaeuser <muesli@tomahawk-player.org>
* *
* Tomahawk is free software: you can redistribute it and/or modify * Tomahawk is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by * it under the terms of the GNU General Public License as published by
@@ -26,6 +26,7 @@
#include <QMutex> #include <QMutex>
#include "Query.h" #include "Query.h"
#include "DatabaseCommand_UpdateSearchIndex.h"
namespace lucene namespace lucene
{ {
@@ -60,7 +61,7 @@ public:
void beginIndexing(); void beginIndexing();
void endIndexing(); void endIndexing();
void appendFields( const QMap< unsigned int, QMap< QString, QString > >& trackData ); void appendFields( const IndexData& data );
signals: signals:
void indexReady(); void indexReady();
@@ -82,6 +83,7 @@ private:
lucene::analysis::SimpleAnalyzer* m_analyzer; lucene::analysis::SimpleAnalyzer* m_analyzer;
lucene::store::Directory* m_luceneDir; lucene::store::Directory* m_luceneDir;
lucene::index::IndexReader* m_luceneReader; lucene::index::IndexReader* m_luceneReader;
lucene::index::IndexWriter* m_luceneWriter;
lucene::search::IndexSearcher* m_luceneSearcher; lucene::search::IndexSearcher* m_luceneSearcher;
}; };