mirror/tomahawk
1
0
Fork 0
mirror of https://github.com/tomahawk-player/tomahawk.git synced 2025-04-21 00:12:06 +02:00
Code Issues Releases Wiki Activity

* Worked around Lucene memory leak and improved our own memory foot-print during indexing.

Browse Source
This commit is contained in:
Christian Muehlhaeuser 2013-05-07 04:23:46 +02:00
parent c04fde0727
commit 64cf40ce8f
4 changed files with 65 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
src/libtomahawk/database/DatabaseCommand_UpdateSearchIndex.cpp
View File

@ -1,6 +1,6 @@
/* === This file is part of Tomahawk Player - <http://tomahawk-player.org> ===
*
* Copyright 2010-2011, Christian Muehlhaeuser <muesli@tomahawk-player.org>
* Copyright 2010-2013, Christian Muehlhaeuser <muesli@tomahawk-player.org>
* Copyright 2012 Leo Franchi <lfranchi@kde.org>
*
* Tomahawk is free software: you can redistribute it and/or modify
@ -49,6 +49,8 @@ DatabaseCommand_UpdateSearchIndex::DatabaseCommand_UpdateSearchIndex()
DatabaseCommand_UpdateSearchIndex::~DatabaseCommand_UpdateSearchIndex()
{
tDebug() << Q_FUNC_INFO;
#ifndef ENABLE_HEADLESS
if ( ! m_statusJob.isNull() )
m_statusJob.data()->done();
@ -61,35 +63,30 @@ DatabaseCommand_UpdateSearchIndex::exec( DatabaseImpl* db )
{
db->m_fuzzyIndex->beginIndexing();
QMap< unsigned int, QMap< QString, QString > > data;
TomahawkSqlQuery q = db->newquery();
q.exec( "SELECT track.id, track.name, artist.name, artist.id FROM track, artist WHERE artist.id = track.artist" );
while ( q.next() )
{
QMap< QString, QString > track;
track.insert( "track", q.value( 1 ).toString() );
track.insert( "artist", q.value( 2 ).toString() );
track.insert( "artistid", q.value( 3 ).toString() );
IndexData ida;
ida.id = q.value( 0 ).toUInt();
ida.artistId = q.value( 3 ).toUInt();
ida.track = q.value( 1 ).toString();
ida.artist = q.value( 2 ).toString();
data.insert( q.value( 0 ).toUInt(), track );
db->m_fuzzyIndex->appendFields( ida );
}
db->m_fuzzyIndex->appendFields( data );
data.clear();
q.exec( "SELECT album.id, album.name FROM album" );
while ( q.next() )
{
QMap< QString, QString > album;
album.insert( "album", q.value( 1 ).toString() );
IndexData ida;
ida.id = q.value( 0 ).toUInt();
ida.album = q.value( 1 ).toString();
data.insert( q.value( 0 ).toUInt(), album );
db->m_fuzzyIndex->appendFields( ida );
}
db->m_fuzzyIndex->appendFields( data );
qDebug() << "Building index finished.";
tDebug( LOGVERBOSE ) << "Building index finished.";
db->m_fuzzyIndex->endIndexing();
}

11
src/libtomahawk/database/DatabaseCommand_UpdateSearchIndex.h
View File

@ -1,6 +1,6 @@
/* === This file is part of Tomahawk Player - <http://tomahawk-player.org> ===
*
* Copyright 2010-2011, Christian Muehlhaeuser <muesli@tomahawk-player.org>
* Copyright 2010-2013, Christian Muehlhaeuser <muesli@tomahawk-player.org>
*
* Tomahawk is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@ -25,6 +25,15 @@
class IndexingJobItem;
struct IndexData
{
unsigned int id;
unsigned int artistId;
QString artist;
QString album;
QString track;
};
class DLLEXPORT DatabaseCommand_UpdateSearchIndex : public DatabaseCommand
{
Q_OBJECT

82
src/libtomahawk/database/FuzzyIndex.cpp
View File

@ -1,6 +1,6 @@
/* === This file is part of Tomahawk Player - <http://tomahawk-player.org> ===
*
* Copyright 2010-2011, Christian Muehlhaeuser <muesli@tomahawk-player.org>
* Copyright 2010-2013, Christian Muehlhaeuser <muesli@tomahawk-player.org>
*
* Tomahawk is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@ -24,7 +24,6 @@
#include <CLucene.h>
#include <CLucene/queryParser/MultiFieldQueryParser.h>
#include "DatabaseCommand_UpdateSearchIndex.h"
#include "DatabaseImpl.h"
#include "Database.h"
#include "utils/TomahawkUtils.h"
@ -112,10 +111,11 @@ FuzzyIndex::beginIndexing()
try
{
qDebug() << Q_FUNC_INFO << "Starting indexing.";
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "Starting indexing.";
if ( m_luceneReader != 0 )
{
qDebug() << "Deleting old lucene stuff.";
tDebug( LOGVERBOSE ) << "Deleting old lucene stuff.";
m_luceneSearcher->close();
m_luceneReader->close();
delete m_luceneSearcher;
@ -124,8 +124,8 @@ FuzzyIndex::beginIndexing()
m_luceneReader = 0;
}
qDebug() << "Creating new index writer.";
IndexWriter luceneWriter( m_luceneDir, m_analyzer, true );
tDebug( LOGVERBOSE ) << "Creating new index writer.";
m_luceneWriter = new IndexWriter( m_luceneDir, m_analyzer, true );
}
catch( CLuceneError& error )
{
@ -138,62 +138,52 @@ FuzzyIndex::beginIndexing()
void
FuzzyIndex::endIndexing()
{
m_luceneWriter->optimize();
m_luceneWriter->close();
delete m_luceneWriter;
m_luceneWriter = 0;
m_mutex.unlock();
emit indexReady();
}
void
FuzzyIndex::appendFields( const QMap< unsigned int, QMap< QString, QString > >& trackData )
FuzzyIndex::appendFields( const IndexData& data )
{
try
{
tDebug() << "Appending to index:" << trackData.count();
bool create = !IndexReader::indexExists( TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" ).toStdString().c_str() );
IndexWriter luceneWriter( m_luceneDir, m_analyzer, create );
Document doc;
QMapIterator< unsigned int, QMap< QString, QString > > it( trackData );
while ( it.hasNext() )
if ( !data.track.isEmpty() )
{
it.next();
unsigned int id = it.key();
QMap< QString, QString > values = it.value();
doc.add( *( _CLNEW Field( _T( "fulltext" ), DatabaseImpl::sortname( QString( "%1 %2" ).arg( data.artist ).arg( data.track ) ).toStdWString().c_str(),
Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) );
if ( values.contains( "track" ) )
{
doc.add( *( _CLNEW Field( _T( "fulltext" ), DatabaseImpl::sortname( QString( "%1 %2" ).arg( values.value( "artist" ) ).arg( values.value( "track" ) ) ).toStdWString().c_str(),
Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) );
doc.add( *( _CLNEW Field( _T( "track" ), DatabaseImpl::sortname( data.track ).toStdWString().c_str(),
Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) );
doc.add( *( _CLNEW Field( _T( "track" ), DatabaseImpl::sortname( values.value( "track" ) ).toStdWString().c_str(),
Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) );
doc.add( *( _CLNEW Field( _T( "artist" ), DatabaseImpl::sortname( data.artist ).toStdWString().c_str(),
Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) );
doc.add( *( _CLNEW Field( _T( "artist" ), DatabaseImpl::sortname( values.value( "artist" ) ).toStdWString().c_str(),
Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) );
doc.add( *( _CLNEW Field( _T( "artistid" ), QString::number( data.artistId ).toStdWString().c_str(),
Field::STORE_YES | Field::INDEX_NO ) ) );
doc.add( *( _CLNEW Field( _T( "artistid" ), values.value( "artistid" ).toStdWString().c_str(),
Field::STORE_YES | Field::INDEX_NO ) ) );
doc.add( *( _CLNEW Field( _T( "trackid" ), QString::number( id ).toStdWString().c_str(),
Field::STORE_YES | Field::INDEX_NO ) ) );
}
else if ( values.contains( "album" ) )
{
doc.add( *( _CLNEW Field( _T( "album" ), DatabaseImpl::sortname( values.value( "album" ) ).toStdWString().c_str(),
Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) );
doc.add( *( _CLNEW Field( _T( "albumid" ), QString::number( id ).toStdWString().c_str(),
Field::STORE_YES | Field::INDEX_NO ) ) );
}
else
Q_ASSERT( false );
luceneWriter.addDocument( &doc );
doc.clear();
doc.add( *( _CLNEW Field( _T( "trackid" ), QString::number( data.id ).toStdWString().c_str(),
Field::STORE_YES | Field::INDEX_NO ) ) );
}
else if ( !data.album.isEmpty() )
{
doc.add( *( _CLNEW Field( _T( "album" ), DatabaseImpl::sortname( data.album ).toStdWString().c_str(),
Field::STORE_NO | Field::INDEX_UNTOKENIZED ) ) );
luceneWriter.optimize();
luceneWriter.close();
doc.add( *( _CLNEW Field( _T( "albumid" ), QString::number( data.id ).toStdWString().c_str(),
Field::STORE_YES | Field::INDEX_NO ) ) );
}
else
Q_ASSERT( false );
m_luceneWriter->addDocument( &doc );
}
catch( CLuceneError& error )
{
@ -223,7 +213,7 @@ FuzzyIndex::search( const Tomahawk::query_ptr& query )
{
if ( !IndexReader::indexExists( TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" ).toStdString().c_str() ) )
{
qDebug() << Q_FUNC_INFO << "index didn't exist.";
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "index didn't exist.";
return resultsmap;
}
@ -314,7 +304,7 @@ FuzzyIndex::searchAlbum( const Tomahawk::query_ptr& query )
{
if ( !IndexReader::indexExists( TomahawkUtils::appDataDir().absoluteFilePath( "tomahawk.lucene" ).toStdString().c_str() ) )
{
qDebug() << Q_FUNC_INFO << "index didn't exist.";
tDebug( LOGVERBOSE ) << Q_FUNC_INFO << "index didn't exist.";
return resultsmap;
}

8
src/libtomahawk/database/FuzzyIndex.h
View File

@ -1,6 +1,6 @@
/* === This file is part of Tomahawk Player - <http://tomahawk-player.org> ===
*
* Copyright 2010-2011, Christian Muehlhaeuser <muesli@tomahawk-player.org>
*
* Copyright 2010-2013, Christian Muehlhaeuser <muesli@tomahawk-player.org>
*
* Tomahawk is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@ -26,6 +26,7 @@
#include <QMutex>
#include "Query.h"
#include "DatabaseCommand_UpdateSearchIndex.h"
namespace lucene
{
@ -60,7 +61,7 @@ public:
void beginIndexing();
void endIndexing();
void appendFields( const QMap< unsigned int, QMap< QString, QString > >& trackData );
void appendFields( const IndexData& data );
signals:
void indexReady();
@ -82,6 +83,7 @@ private:
lucene::analysis::SimpleAnalyzer* m_analyzer;
lucene::store::Directory* m_luceneDir;
lucene::index::IndexReader* m_luceneReader;
lucene::index::IndexWriter* m_luceneWriter;
lucene::search::IndexSearcher* m_luceneSearcher;
};