File oss-moodle-calender-310.patch of Package oss-moodle

Overview Repositories Revisions Requests Users Attributes Meta

File oss-moodle-calender-310.patch of Package oss-moodle (Revision 413f8a5bfb2ee9b74ae45c830b998b80)

Currently displaying revision 413f8a5bfb2ee9b74ae45c830b998b80 , Show latest

From f66047fb084be7aaeb667f301a8c14b1bb37fa1e Mon Sep 17 00:00:00 2001
From: Andreas Grabs <moodle@grabs-edv.de>
Date: Mon, 11 Jan 2021 08:08:30 +0100
Subject: [PATCH] core_calendar: add indices to the events table

Add some indices to the "events" table to improve query performance in
raw_event_retrieval_strategy::get_raw_events_legacy_implementation().

This will likely only affect MySQL / MariaDB installations.

It also moves the `WHERE` statement from the `$unionstartquery` to
the individual subqueries in the `$subqueryconditions` array. This
change should preserve the query (modulo whitespace changes), since the
`$unionstartquery` variable is prepended as well as used as a joiner
when imploding the `$subqueryconditions`.

On mysql-based db systems, the queries built by `raw_event_retrieval_strategy`
are supplemented by adding a `FORCE INDEX` statements between the `FROM`
clause and the `WHERE` clause.

This significantly improves performance on instances with a large number of
rows in the events table (we observed improvements from ~4m to ~500ms).

This patch has not been tested on PostgreSQL systems.

Co-authored-by: Stefan Auditor <stefan@auditor.email>
Co-authored-by: Fabian Freyer <fabian.freyer@physik.tu-berlin.de>
---
 .../raw_event_retrieval_strategy.php          | 98 ++++++++++++++++---
 lib/db/install.xml                            |  5 +
 lib/db/upgrade.php                            | 52 ++++++++++
 3 files changed, 143 insertions(+), 12 deletions(-)

diff --git a/calendar/classes/local/event/strategies/raw_event_retrieval_strategy.php b/calendar/classes/local/event/strategies/raw_event_retrieval_strategy.php
index 46b42d7899cf..6b609ea6c984 100644
--- a/calendar/classes/local/event/strategies/raw_event_retrieval_strategy.php
+++ b/calendar/classes/local/event/strategies/raw_event_retrieval_strategy.php
@@ -203,7 +203,7 @@ protected function get_raw_events_legacy_implementation(
                     // Set filter condition for the user's events.
                     // Even though $user is a single scalar, we still use get_in_or_equal() because we are inside a loop.
                     list($inusers, $inuserparams) = $DB->get_in_or_equal($user, SQL_PARAMS_NAMED);
-                    $subqueryconditions[] = "(ev.userid $inusers AND ev.courseid = 0 AND ev.groupid = 0 AND ev.categoryid = 0)";
+                    $subqueryconditions[] = "WHERE (ev.userid $inusers AND ev.courseid = 0 AND ev.groupid = 0 AND ev.categoryid = 0)";
                     $subqueryparams = array_merge($subqueryparams, $inuserparams);
 
                     foreach ($usercourses as $courseid) {
@@ -220,31 +220,31 @@ protected function get_raw_events_legacy_implementation(
                 // Set filter condition for the user's group events.
                 if ($usergroups === true || $viewgroupsonly) {
                     // Fetch group events, but not group overrides.
-                    $subqueryconditions[] = "(ev.groupid != 0 AND ev.eventtype = 'group')";
+                    $subqueryconditions[] = "WHERE (ev.groupid != 0 AND ev.eventtype = 'group')";
                 } else if (!empty($usergroups)) {
                     // Fetch group events and group overrides.
                     list($inusergroups, $inusergroupparams) = $DB->get_in_or_equal($usergroups, SQL_PARAMS_NAMED);
-                    $subqueryconditions[] = "(ev.groupid $inusergroups)";
+                    $subqueryconditions[] = "WHERE (ev.groupid $inusergroups)";
                     $subqueryparams = array_merge($subqueryparams, $inusergroupparams);
                 }
             }
         } else if ($users === true) {
             // Events from ALL users.
-            $subqueryconditions[] = "(ev.userid != 0 AND ev.courseid = 0 AND ev.groupid = 0 AND ev.categoryid = 0)";
+            $subqueryconditions[] = "WHERE (ev.userid != 0 AND ev.courseid = 0 AND ev.groupid = 0 AND ev.categoryid = 0)";
 
             if (is_array($groups)) {
                 // Events from a number of groups.
                 list($insqlgroups, $inparamsgroups) = $DB->get_in_or_equal($groups, SQL_PARAMS_NAMED);
-                $subqueryconditions[] = "ev.groupid $insqlgroups";
+                $subqueryconditions[] = "WHERE ev.groupid $insqlgroups";
                 $subqueryparams = array_merge($subqueryparams, $inparamsgroups);
             } else if ($groups === true) {
                 // Events from ALL groups.
-                $subqueryconditions[] = "ev.groupid != 0";
+                $subqueryconditions[] = "WHERE ev.groupid != 0";
             }
 
             if ($courses === true) {
                 // ALL course events. It's not needed to worry about users' access as $users = true.
-                $subqueryconditions[] = "(ev.groupid = 0 AND ev.courseid != 0 AND ev.categoryid = 0)";
+                $subqueryconditions[] = "WHERE (ev.groupid = 0 AND ev.courseid != 0 AND ev.categoryid = 0)";
             }
         }
 
@@ -260,27 +260,32 @@ protected function get_raw_events_legacy_implementation(
             $subquerycourses = array_unique($subquerycourses);
         }
 
+        // On MySQL-based databases, the optimizer does not automatically
+        // use the indices in certain cases. Build an index hint for these.
+        $forceindexhint = self::index_hint_sql(array('groupid', 'categoryid', 'courseid'));
+
         // Set subquery filter condition for the courses.
         if (!empty($subquerycourses)) {
             list($incourses, $incoursesparams) = $DB->get_in_or_equal($subquerycourses, SQL_PARAMS_NAMED);
-            $subqueryconditions[] = "(ev.groupid = 0 AND ev.courseid $incourses AND ev.categoryid = 0)";
+            $subqueryconditions[] = "$forceindexhint
+                                     WHERE (ev.groupid = 0 AND ev.courseid $incourses AND ev.categoryid = 0)";
             $subqueryparams = array_merge($subqueryparams, $incoursesparams);
         }
 
         // Set subquery filter condition for the categories.
         if ($categories === true) {
-            $subqueryconditions[] = "(ev.categoryid != 0 AND ev.eventtype = 'category')";
+            $subqueryconditions[] = "WHERE (ev.categoryid != 0 AND ev.eventtype = 'category')";
         } else if (!empty($categories)) {
             list($incategories, $incategoriesparams) = $DB->get_in_or_equal($categories, SQL_PARAMS_NAMED);
-            $subqueryconditions[] = "(ev.groupid = 0 AND ev.courseid = 0 AND ev.categoryid $incategories)";
+            $subqueryconditions[] = "$forceindexhint
+                                     WHERE (ev.groupid = 0 AND ev.courseid = 0 AND ev.categoryid $incategories)";
             $subqueryparams = array_merge($subqueryparams, $incategoriesparams);
         }
 
         // Build the WHERE condition for the sub-query.
         if (!empty($subqueryconditions)) {
             $unionstartquery = "SELECT modulename, instance, eventtype, priority
-                                  FROM {event} ev
-                                 WHERE ";
+                                FROM {event} ev ";
             $subqueryunion = '('.$unionstartquery . implode(" UNION $unionstartquery ", $subqueryconditions).')';
         } else {
             $subqueryunion = '{event}';
@@ -299,9 +304,13 @@ protected function get_raw_events_legacy_implementation(
                        FROM $subqueryunion ev
                    GROUP BY ev.modulename, ev.instance, ev.eventtype";
 
+        // Build the index hint for mysql based databases.
+        $useindexhint = self::index_hint_sql(array('groupid', 'courseid', 'categoryid', 'visible', 'userid'), 'USE');
+
         // Build the main query.
         $sql = "SELECT e.*
                   FROM {event} e
+                  $useindexhint
             INNER JOIN ($subquery) fe
                     ON e.modulename = fe.modulename
                        AND e.instance = fe.instance
@@ -320,4 +329,69 @@ protected function get_raw_events_legacy_implementation(
 
         return  $events === false ? [] : $events;
     }
+
+    /**
+     * Add some indexes to the "events" table to improve query performance.
+     * Note: Only mysql based databases are supported!
+     * This method build the sql part with the use|force index statement.
+     *
+     * @param array $columns The columns used in this index.
+     * @param string $hinttype The type "USE" or "FORCE"
+     * @param string $table The xmldb table name
+     * @param bool $uniqueindex Index is unique true|false
+     * @return string The sql part for usage in query
+     */
+    public static function index_hint_sql($columns, $hinttype = 'FORCE', $table = 'event', $uniqueindex = false) {
+        global $DB;
+
+        // Only mysql based databases are supported!
+        if ($DB->get_dbfamily() != 'mysql') {
+            return '';
+        }
+
+        $indexname = self::get_indexname($columns, $table, $uniqueindex);
+        if (empty($indexname)) {
+            // Index was not found so we return an empty string.
+            return '';
+        }
+
+        $hinttype = strtoupper($hinttype);
+        $sqlpart = "$hinttype INDEX ($indexname)";
+
+        return $sqlpart;
+    }
+
+    /**
+     * Add some indexes to the "events" table to improve query performance.
+     * Note: Only mysql based databases are supported!
+     * This method find an index name by the columns definition.
+     *
+     * @param array $columns The columns used in this index.
+     * @param string $table The xmldb table name
+     * @param bool $uniqueindex Index is unique true|false
+     * @return string The real name of the index
+     */
+    private static function get_indexname($columns, $table = 'event', $uniqueindex = false) {
+        global $DB;/** @var \moodle_database $DB */
+
+        // We don't want to look into this on every call.
+        static $indexes;
+
+        if (empty($indexes)) {
+            $indexes = $DB->get_indexes($table);
+        }
+
+        if (is_array($indexes)) {
+            $search = array(
+                'unique' => $uniqueindex,
+                'columns' => $columns
+            );
+            foreach ($indexes as $key => $index) {
+                if ($index == $search) {
+                    return $key;
+                }
+            }
+        }
+        return '';
+    }
 }
diff --git a/lib/db/install.xml b/lib/db/install.xml
index 87a8826c2fd2..b6fa3fc661d2 100644
--- a/lib/db/install.xml
+++ b/lib/db/install.xml
@@ -467,6 +467,11 @@
         <INDEX NAME="eventtype" UNIQUE="false" FIELDS="eventtype"/>
         <INDEX NAME="component" UNIQUE="false" FIELDS="component, eventtype, instance"/>
         <INDEX NAME="modulename-instance-eventtype" UNIQUE="false" FIELDS="modulename, instance, eventtype" COMMENT="Improve performance on assign adhoc tasks."/>
+        <INDEX NAME="type-groupid-courseid-visible-timesort" UNIQUE="false" FIELDS="type, groupid, courseid, visible, timesort"/>
+        <INDEX NAME="userid-courseid-groupid-categoryid" UNIQUE="false" FIELDS="userid, courseid, groupid, categoryid"/>
+        <INDEX NAME="groupid-categoryid-courseid" UNIQUE="false" FIELDS="groupid, categoryid, courseid"/>
+        <INDEX NAME="eventtype-categoryid" UNIQUE="false" FIELDS="eventtype, categoryid"/>
+        <INDEX NAME="groupid-courseid" UNIQUE="false" FIELDS="groupid, courseid"/>
       </INDEXES>
     </TABLE>
     <TABLE NAME="cache_filters" COMMENT="For keeping information about cached data">
diff --git a/lib/db/upgrade.php b/lib/db/upgrade.php
index 25e5433498dd..26df391e0bff 100644
--- a/lib/db/upgrade.php
+++ b/lib/db/upgrade.php
@@ -3014,6 +3014,58 @@ function xmldb_main_upgrade($oldversion) {
             }
         } while ($invalidconfigrations);
 
+        // Add some indexes to the "events" table to improve query performance in
+        // \core_calendar\local\event\strategies\raw_event_retrieval_strategy::get_raw_events_legacy_implementation()
+        // This will likely only affect MySQL / MariaDB installations.
+        // Define table event.
+        $table = new \xmldb_table('event');
+
+        // Add indexes.
+        $index = new xmldb_index(
+            'type-groupid-courseid-visible-timesort',
+            XMLDB_INDEX_NOTUNIQUE,
+            array('type', 'groupid', 'courseid', 'visible', 'timesort')
+        );
+        if (!$dbman->index_exists($table, $index)) {
+            $dbman->add_index($table, $index);
+        }
+
+        $index = new xmldb_index(
+            'userid-courseid-groupid-categoryid',
+            XMLDB_INDEX_NOTUNIQUE,
+            array('userid','courseid', 'groupid', 'categoryid')
+        );
+        if (!$dbman->index_exists($table, $index)) {
+            $dbman->add_index($table, $index);
+        }
+
+        $index = new xmldb_index(
+            'groupid-categoryid-courseid',
+            XMLDB_INDEX_NOTUNIQUE,
+            array('groupid', 'categoryid', 'courseid')
+        );
+        if (!$dbman->index_exists($table, $index)) {
+            $dbman->add_index($table, $index);
+        }
+
+        $index = new xmldb_index(
+            'eventtype-categoryid',
+            XMLDB_INDEX_NOTUNIQUE,
+            array('eventtype', 'categoryid')
+        );
+        if (!$dbman->index_exists($table, $index)) {
+            $dbman->add_index($table, $index);
+        }
+
+        $index = new xmldb_index(
+            'groupid-courseid',
+            XMLDB_INDEX_NOTUNIQUE,
+            array('groupid', 'courseid')
+        );
+        if (!$dbman->index_exists($table, $index)) {
+            $dbman->add_index($table, $index);
+        }
+
         upgrade_main_savepoint(true, 2020110900.01);
     }

Places

File oss-moodle-calender-310.patch of Package oss-moodle (Revision 413f8a5bfb2ee9b74ae45c830b998b80)

Places