File fileindexer-do-not-use-QDirIterators-in-the-que... of Package nepomuk-core

Overview Repositories Revisions Requests Users Attributes Meta

File fileindexer-do-not-use-QDirIterators-in-the-queues.patch of Package nepomuk-core (Revision f463790f55a5639caf571f651a10bc5f)

Currently displaying revision f463790f55a5639caf571f651a10bc5f , Show latest

From 630939e388beb4eaf6387a4f9bb827a87fdd0478 Mon Sep 17 00:00:00 2001
From: Vishesh Handa <me@vhanda.in>
Date: Thu, 14 Feb 2013 13:10:05 +0530
Subject: [PATCH 1/5] FileIndexer: Do not use QDirIterators in the queues

A DirIterator works by opening the directory (as a file) and reading its
contents, which are the list of files in that directory. As long as the
dirIterator is open, it consumes one file descriptor.

When iterating over all the folders, by using QDirIterator, we open a
new file descriptor per directory. Since we traverse the file system via
breadth first, the number of valid QDirIterators can get quite high.
This sometimes results in us exceeding the number of file descriptors
required which results in a crash, cause QProcess doesn't handle running
out of file descriptors.

Instead we now just iterate over all the directories and store them as
strings. This may consume more memory, but it is better than a crash.

BUG: 310777
REVIEW: 108950
---
 services/fileindexer/basicindexingqueue.cpp | 39 +++++------------------------
 services/fileindexer/basicindexingqueue.h   |  1 -
 2 files changed, 6 insertions(+), 34 deletions(-)

diff --git a/services/fileindexer/basicindexingqueue.cpp b/services/fileindexer/basicindexingqueue.cpp
index b581786..074d021 100644
--- a/services/fileindexer/basicindexingqueue.cpp
+++ b/services/fileindexer/basicindexingqueue.cpp
@@ -44,12 +44,6 @@ void BasicIndexingQueue::clear()
     m_currentUrl.clear();
     m_currentFlags = NoUpdateFlags;
     m_paths.clear();
-
-    typedef QPair<QDirIterator*, UpdateDirFlags> DirPair;
-    foreach( const DirPair& pair, m_iterators )
-        delete pair.first;
-
-    m_iterators.clear();
 }
 
 void BasicIndexingQueue::clear(const QString& path)
@@ -60,16 +54,6 @@ void BasicIndexingQueue::clear(const QString& path)
         if( it.value().first.startsWith( path ) )
             it.remove();
     }
-
-    QMutableListIterator< QPair<QDirIterator*, UpdateDirFlags> > iter( m_iterators );
-    while( iter.hasNext() ) {
-        QDirIterator* dirIter =  iter.next().first;
-
-        if( dirIter->path().startsWith( path ) ) {
-            iter.remove();
-            delete dirIter;
-        }
-    }
 }
 
 QUrl BasicIndexingQueue::currentUrl() const
@@ -85,7 +69,7 @@ UpdateDirFlags BasicIndexingQueue::currentFlags() const
 
 bool BasicIndexingQueue::isEmpty()
 {
-    return m_iterators.isEmpty() && m_paths.isEmpty();
+    return m_paths.isEmpty();
 }
 
 void BasicIndexingQueue::enqueue(const QString& path)
@@ -111,20 +95,7 @@ void BasicIndexingQueue::processNextIteration()
 {
     bool processingFile = false;
 
-    // First process all the iterators and then the paths
-    if( !m_iterators.isEmpty() ) {
-        QPair< QDirIterator*, UpdateDirFlags > pair = m_iterators.first();
-        QDirIterator* dirIt = pair.first;
-
-        if( dirIt->hasNext() ) {
-            processingFile = process( dirIt->next(), pair.second );
-        }
-        else {
-            delete m_iterators.dequeue().first;
-        }
-    }
-
-    else if( !m_paths.isEmpty() ) {
+    if( !m_paths.isEmpty() ) {
         QPair< QString, UpdateDirFlags > pair = m_paths.dequeue();
         processingFile = process( pair.first, pair.second );
     }
@@ -160,8 +131,10 @@ bool BasicIndexingQueue::process(const QString& path, UpdateDirFlags flags)
         if( recursive && !info.isSymLink() && shouldIndexContents(path) ) {
             QDir::Filters dirFilter = QDir::NoDotAndDotDot|QDir::Readable|QDir::Files|QDir::Dirs;
 
-            QPair<QDirIterator*, UpdateDirFlags> pair = qMakePair( new QDirIterator( path, dirFilter ), flags );
-            m_iterators.enqueue( pair );
+            QDirIterator it( path, dirFilter );
+            while( it.hasNext() ) {
+                m_paths.enqueue( qMakePair(it.next(), flags) );
+            }
         }
     }
     else if( info.isFile() && (forced || indexingRequired) ) {
diff --git a/services/fileindexer/basicindexingqueue.h b/services/fileindexer/basicindexingqueue.h
index 5d1c190..ca02996 100644
--- a/services/fileindexer/basicindexingqueue.h
+++ b/services/fileindexer/basicindexingqueue.h
@@ -106,7 +106,6 @@ namespace Nepomuk2 {
         bool process(const QString& path, Nepomuk2::UpdateDirFlags flags);
 
         QQueue< QPair<QString, UpdateDirFlags> > m_paths;
-        QQueue< QPair<QDirIterator*, UpdateDirFlags> > m_iterators;
 
         QUrl m_currentUrl;
         QString m_currentMimeType;
-- 
1.8.1.2

Places

File fileindexer-do-not-use-QDirIterators-in-the-queues.patch of Package nepomuk-core (Revision f463790f55a5639caf571f651a10bc5f)

Places