Merge pull request #35 from statusengine/deadlock

Restart transaction on MySQL deadlocks
statusengine · May 25, 2022 · 0173e0e · 0173e0e
2 parents 0e388e3 + 58227fd
commit 0173e0e
Show file tree

Hide file tree

Showing 17 changed files with 67 additions and 47 deletions.
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-3.7.4
+3.7.5
diff --git a/src/Backends/MySQL/MySQL.php b/src/Backends/MySQL/MySQL.php
@@ -280,31 +280,51 @@ public function dispatch() {
      * @param \PDOStatement $query
      * @return bool
      * @throws StorageBackendUnavailableExceptions
-     */
-    public function executeQuery(\PDOStatement $query) {
-        $result = false;
-        try {
-            $result = $query->execute();
-
-        } catch (\Exception $Exception) {
-            $errorNo = $Exception->errorInfo[1];
-            $errorString = $Exception->errorInfo[2];
-            $this->Syslog->error(sprintf('[%s] %s', $errorNo, $errorString));
-            $this->Syslog->error($query->queryString);
-            $this->Syslog->error("Run the worker in foreground mode to see the full query: https://statusengine.org/worker/#debugging");
-
-            // This function has no return - so no log file -.-
-            // If Statusengine is running via systemd systemd will write the messages to syslog
-            if ($this->isDumpOfMysqlQueryParametersEnabled) {
-                $query->debugDumpParams();
-            }
-
-            if ($errorString == 'MySQL server has gone away') {
-                $this->reconnect();
-                throw new StorageBackendUnavailableExceptions($errorString);
+     * @package string $caller
+     */
+    public function executeQuery(\PDOStatement $query, $caller = 'Unknown') {
+        // https://dev.mysql.com/doc/refman/8.0/en/innodb-deadlocks.html
+        // https://dev.mysql.com/doc/refman/8.0/en/innodb-deadlocks-handling.html
+        // The deadlock logic is ported from Statusengine 2
+        // https://github.com/nook24/statusengine/blame/b5c86f0e02fd69a7045eb652f49dad20ee4d2b67/cakephp/app/src/BulkRepository.php#L185
+        $retries = 10;
+        for ($i = 1; $i < $retries; $i++) {
+            try {
+                return $query->execute();
+
+            } catch (\Exception $Exception) {
+                $sqlstateErrorCode = $Exception->errorInfo[0]; // SQLSTATE error code (a five characters alphanumeric identifier defined in the ANSI SQL standard).
+                $errorNo = $Exception->errorInfo[1]; //  Driver-specific error code.
+                $errorString = $Exception->errorInfo[2]; //  Driver-specific error message.
+                $this->Syslog->error(sprintf('[%s] %s', $errorNo, $errorString));
+
+                if ($i <= $retries && $sqlstateErrorCode == 40001 && $errorNo == 1213) {
+                    // This is a InnoDB deadlock - retry
+                    $sleep = 50000 + rand(0, 450000);
+                    $this->Syslog->info('Encountered MySQL Deadlock during transaction on ' . $caller . '. Retry transaction in ' . floor($sleep / 1000) . 'ms (try ' . ($i) . '/' . $retries . ')');
+                    usleep($sleep);
+                } else if ($sqlstateErrorCode == 40001 && $errorNo == 1213) {
+                    // too many deadlocks
+                    $this->Syslog->info('Couldn\'t solve deadlock for ' . $caller . '. Ignore for now to prevent crash: Exception: ' . $Exception->getMessage());
+                } else {
+                    // Any other error
+                    $this->Syslog->error($query->queryString);
+                    $this->Syslog->error("Run the worker in foreground mode to see the full query: https://statusengine.org/worker/#debugging");
+
+                    // This function has no return - so no log file -.-
+                    // If Statusengine is running via systemd systemd will write the messages to syslog
+                    if ($this->isDumpOfMysqlQueryParametersEnabled) {
+                        $query->debugDumpParams();
+                    }
+
+                    if ($errorString == 'MySQL server has gone away') {
+                        $this->reconnect();
+                        throw new StorageBackendUnavailableExceptions($errorString);
+                    }
+                }
             }
         }
-        return $result;
+        return false;
     }
 
     /**

diff --git a/src/Backends/MySQL/SqlObjects/MysqlHostAcknowledgement.php b/src/Backends/MySQL/SqlObjects/MysqlHostAcknowledgement.php
@@ -78,7 +78,7 @@ public function insert($isRecursion = false) {
         $query->bindValue($i++, (int)$this->Acknowledgement->isNotifyContacts());
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlHostAcknowledgement');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {

diff --git a/src/Backends/MySQL/SqlObjects/MysqlHostDowntimehistory.php b/src/Backends/MySQL/SqlObjects/MysqlHostDowntimehistory.php
@@ -83,7 +83,7 @@ public function saveDowntime(Downtime $Downtime, $isRecursion = false) {
         }
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MySQLHostDowntimehistory');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {
@@ -108,7 +108,7 @@ public function deleteDowntime(Downtime $Downtime, $isRecursion = false) {
         $query->bindValue(4, $Downtime->getDowntimeId());
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MySQLHostDowntimehistory');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {

diff --git a/src/Backends/MySQL/SqlObjects/MysqlHostScheduleddowntime.php b/src/Backends/MySQL/SqlObjects/MysqlHostScheduleddowntime.php
@@ -88,7 +88,7 @@ public function saveDowntime(Downtime $Downtime, $isRecursion = false) {
         }
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlHostScheduleddowntime');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {
@@ -113,7 +113,7 @@ public function deleteDowntime(Downtime $Downtime, $isRecursion = false){
         $query->bindValue(4, $Downtime->getDowntimeId());
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlHostScheduleddowntime');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {

diff --git a/src/Backends/MySQL/SqlObjects/MysqlHostcheck.php b/src/Backends/MySQL/SqlObjects/MysqlHostcheck.php
@@ -93,7 +93,7 @@ public function insert($isRecursion = false){
         }
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlHostcheck');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {

diff --git a/src/Backends/MySQL/SqlObjects/MysqlHoststatus.php b/src/Backends/MySQL/SqlObjects/MysqlHoststatus.php
@@ -126,7 +126,7 @@ public function insert($isRecursion = false){
         }
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlHoststatus');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {
@@ -143,7 +143,7 @@ public function truncate($isRecursion = false){
         $query = $this->MySQL->prepare('DELETE FROM statusengine_hoststatus WHERE node_name=?');
         $query->bindValue(1, $this->nodeName);
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlHoststatus');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {

diff --git a/src/Backends/MySQL/SqlObjects/MysqlLogentry.php b/src/Backends/MySQL/SqlObjects/MysqlLogentry.php
@@ -84,7 +84,7 @@ public function insert($isRecursion = false){
         }
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlLogentry');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {

diff --git a/src/Backends/MySQL/SqlObjects/MysqlNotification.php b/src/Backends/MySQL/SqlObjects/MysqlNotification.php
@@ -129,7 +129,7 @@ public function getHostQuery($hostNotificationCache, $isRecursion = false) {
         }
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlNotification');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {
@@ -169,7 +169,7 @@ public function getServiceQuery($serviceNotificationCache, $isRecursion = false)
         }
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlNotification');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {

diff --git a/src/Backends/MySQL/SqlObjects/MysqlPerfdata.php b/src/Backends/MySQL/SqlObjects/MysqlPerfdata.php
@@ -82,7 +82,7 @@ public function insert($isRecursion = false) {
         }
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlPerfdata');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {

diff --git a/src/Backends/MySQL/SqlObjects/MysqlServiceAcknowledgement.php b/src/Backends/MySQL/SqlObjects/MysqlServiceAcknowledgement.php
@@ -79,7 +79,7 @@ public function insert($isRecursion = false) {
         $query->bindValue($i++, (int)$this->Acknowledgement->isNotifyContacts());
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlServiceAcknowledgement');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {

diff --git a/src/Backends/MySQL/SqlObjects/MysqlServiceDowntimehistory.php b/src/Backends/MySQL/SqlObjects/MysqlServiceDowntimehistory.php
@@ -83,7 +83,7 @@ public function saveDowntime(Downtime $Downtime, $isRecursion = false) {
         }
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlServiceDowntimehistory');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {
@@ -109,7 +109,7 @@ public function deleteDowntime(Downtime $Downtime, $isRecursion = false) {
         $query->bindValue(5, $Downtime->getDowntimeId());
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlServiceDowntimehistory');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {

diff --git a/src/Backends/MySQL/SqlObjects/MysqlServiceScheduleddowntime.php b/src/Backends/MySQL/SqlObjects/MysqlServiceScheduleddowntime.php
@@ -89,7 +89,7 @@ public function saveDowntime(Downtime $Downtime, $isRecursion = false) {
         }
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlServiceScheduleddowntime');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {
@@ -115,7 +115,7 @@ public function deleteDowntime(Downtime $Downtime, $isRecursion = false){
         $query->bindValue(5, $Downtime->getDowntimeId());
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlServiceScheduleddowntime');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {

diff --git a/src/Backends/MySQL/SqlObjects/MysqlServicecheck.php b/src/Backends/MySQL/SqlObjects/MysqlServicecheck.php
@@ -93,7 +93,7 @@ public function insert($isRecursion = false){
         }
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlServicecheck');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {

diff --git a/src/Backends/MySQL/SqlObjects/MysqlServicestatus.php b/src/Backends/MySQL/SqlObjects/MysqlServicestatus.php
@@ -129,7 +129,7 @@ public function insert($isRecursion = false){
         }
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlServicestatus');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {
@@ -146,7 +146,7 @@ public function truncate($isRecursion = false){
         $query = $this->MySQL->prepare('DELETE FROM statusengine_servicestatus WHERE node_name=?');
         $query->bindValue(1, $this->nodeName);
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlServicestatus');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {

diff --git a/src/Backends/MySQL/SqlObjects/MysqlStatechange.php b/src/Backends/MySQL/SqlObjects/MysqlStatechange.php
@@ -129,7 +129,7 @@ public function getHostQuery($hostStatechangeCache, $isRecursion = false) {
         }
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlStatechange');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {
@@ -169,7 +169,7 @@ public function getServiceQuery($serviceStatechangeCache, $isRecursion = false)
         }
 
         try {
-            return $this->MySQL->executeQuery($query);
+            return $this->MySQL->executeQuery($query, 'MysqlStatechange');
         } catch (StorageBackendUnavailableExceptions $Exceptions) {
             //Retry
             if ($isRecursion === false) {

diff --git a/src/Backends/MySQL/SqlObjects/MysqlTask.php b/src/Backends/MySQL/SqlObjects/MysqlTask.php
@@ -90,7 +90,7 @@ public function deleteTaskByUuids($uuids = []) {
             $query->bindValue($i++, $uuid);
         }
 
-        return $this->MySQL->executeQuery($query);
+        return $this->MySQL->executeQuery($query, 'MysqlTask');
     }
 
 }