Exclude node from cluster on non-recoverable error

Author: knizhnik

multimaster.c

@@ -166,7 +166,8 @@ char const* const MtmNodeStatusMnem[] =
 	"Connected",
 	"Online",
 	"Recovery",
-	"InMinor"
+	"InMinor",
+	"OutOfService"
 };
 
 bool  MtmDoReplication;
@@ -1025,8 +1026,11 @@ void MtmHandleApplyError(void)
 	  case ERRCODE_INDEX_CORRUPTED:
 	  case ERRCODE_SYSTEM_ERROR:
 	  case ERRCODE_INTERNAL_ERROR:
-	  case ERRCODE_OUT_OF_MEMORY:
-		break;
+	  case ERRCODE_OUT_OF_MEMORY:		  
+		  elog(WARNING, "Node is excluded from cluster because of non-recoverable error %d", edata->sqlerrcode);
+		  MtmSwitchClusterMode(MTM_OUT_OF_SERVICE);
+		  kill(PostmasterPid, SIGQUIT);
+		  break;
 	}
 }
 

multimaster.h

@@ -97,11 +97,12 @@ typedef enum
 typedef enum
 {
 	MTM_INITIALIZATION, /* Initial status */
-	MTM_OFFLINE,        /* Node is out of quorum */
+	MTM_OFFLINE,        /* Node is excluded from cluster */
 	MTM_CONNECTED,      /* Arbiter is established connections with other nodes */
 	MTM_ONLINE,         /* Ready to receive client's queries */
 	MTM_RECOVERY,       /* Node is in recovery process */
-	MTM_IN_MINORITY     /* Node is out of quorum */
+	MTM_IN_MINORITY,    /* Node is out of quorum */
+	MTM_OUT_OF_SERVICE  /* Node is not avaiable to to critical, non-recoverable error */
 } MtmNodeStatus;
 
 typedef enum