Store last successful MS host for agent reconnection fallback

agent/src/main/java/com/cloud/agent/Agent.java

@@ -91,7 +91,6 @@
 import com.cloud.utils.nio.HandlerFactory;
 import com.cloud.utils.nio.Link;
 import com.cloud.utils.nio.NioClient;
-import com.cloud.utils.nio.NioConnection;
 import com.cloud.utils.nio.Task;
 import com.cloud.utils.script.Script;
 
@@ -131,7 +130,7 @@ public int value() {
     CopyOnWriteArrayList<IAgentControlListener> controlListeners = new CopyOnWriteArrayList<>();
 
     IAgentShell shell;
-    NioConnection connection;
+    NioClient connection;
     ServerResource serverResource;
     Link link;
     Long id;
@@ -919,6 +918,20 @@ private void processManagementServerList(final List<String> msList, final String
         }
     }
 
+    /**
+     * Saves the currently connected management server host after successful setup completion.
+     * This host is persisted and later added to the reconnection list as a fallback option.
+     * Called after receiving a Ready command from the management server, indicating that
+     * the agent has successfully completed its initialization and is ready to work.
+     *
+     * @param connectedHost the hostname or IP address of the successfully connected management server
+     */
+    private void updateLastSetupCompletedHost(String connectedHost) {
+        if (StringUtils.isNotBlank(connectedHost)) {
+            shell.setLastSetupCompletedHost(connectedHost);
+        }
+    }
+
     private Answer setupManagementServerList(final SetupMSListCommand cmd) {
         processManagementServerList(cmd.getMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval());
         return new SetupMSListAnswer(true);
@@ -959,6 +972,8 @@ public void processReadyCommand(final Command cmd) {
 
         verifyAgentArch(ready.getArch());
         processManagementServerList(ready.getMsHostList(), ready.getLbAlgorithm(), ready.getLbCheckInterval());
+        String connectedHost = shell.getConnectedHost();
+        updateLastSetupCompletedHost(connectedHost);
 
         logger.info("Ready command is processed for agent [id: {}, uuid: {}, name: {}]", getId(), getUuid(), getName());
     }

agent/src/main/java/com/cloud/agent/AgentShell.java

@@ -22,6 +22,7 @@
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Enumeration;
 import java.util.HashMap;
@@ -154,7 +155,27 @@ public void resetHostCounter() {
 
     @Override
     public String[] getHosts() {
-        return _host.split(",");
+        String lastSetupCompletedHost = getLastSetupCompletedHost();
+        String host;
+        // Add the last successful setup host as a fallback option at the end of the host list.
+        // This host is tried only after all configured hosts have failed, providing a
+        // last-resort connection option since this host previously completed setup successfully.
+        if (StringUtils.isNotBlank(lastSetupCompletedHost) && StringUtils.isNotBlank(_host)) {
+            final String candidate = lastSetupCompletedHost.trim();
+            // Match against the exact comma-separated entries so a substring (e.g. 10.0.0.1 in
+            // 10.0.0.10) does not wrongly suppress the fallback.
+            final boolean alreadyPresent = Arrays.stream(_host.split(","))
+                    .map(String::trim)
+                    .anyMatch(candidate::equalsIgnoreCase);
+            host = alreadyPresent ? _host : _host + "," + candidate;
+            if (!alreadyPresent) {
+                LOGGER.warn("Using last successful host [{}] as fallback; it is not in the configured host list [{}]", candidate, _host);
+            }
+        } else {
+            host = _host;
+        }
+
+        return host.split(",");
     }
 
     @Override
@@ -464,6 +485,26 @@ public Integer getSslHandshakeTimeout() {
         return AgentPropertiesFileHandler.getPropertyValue(AgentProperties.SSL_HANDSHAKE_TIMEOUT);
     }
 
+    @Override
+    public void setLastSetupCompletedHost(String host) {
+        if (StringUtils.isNotBlank(host)) {
+            setPersistentProperty(null, AgentProperties.LAST_SETUP_COMPLETED_HOST.getName(), host);
+        }
+    }
+
+    /**
+     * Gets the last host where the agent successfully completed its setup process
+     * and received a Ready command.
+     *
+     * @return the hostname or IP address of the last successfully setup host, or null if none exists
+     */
+    private String getLastSetupCompletedHost() {
+        if (_storage != null) {
+            return getPersistentProperty(null, AgentProperties.LAST_SETUP_COMPLETED_HOST.getName());
+        }
+        return AgentPropertiesFileHandler.getPropertyValue(AgentProperties.LAST_SETUP_COMPLETED_HOST);
+    }
+
     public synchronized int getNextAgentId() {
         return _nextAgentId++;
     }

agent/src/main/java/com/cloud/agent/IAgentShell.java

@@ -72,4 +72,14 @@ public interface IAgentShell {
     void launchNewAgent(ServerResource resource) throws ConfigurationException;
 
     Integer getSslHandshakeTimeout();
+
+    /**
+     * Sets the last host where the agent successfully completed its setup process
+     * and received a Ready command. This value is persisted across agent restarts
+     * and used as a last-resort fallback during reconnection: it is appended after
+     * the configured hosts and tried only once all of them have failed.
+     *
+     * @param host the hostname or IP address where the agent setup completed successfully
+     */
+    void setLastSetupCompletedHost(String host);
 }

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -57,6 +57,14 @@ public class AgentProperties{
      */
     public static final Property<String> HOST = new Property<>("host", "localhost");
 
+    /**
+     * The name of the last host where the agent successfully completed its setup process
+     * and received a Ready command
+     * Data type: String.<br>
+     * Default value: <code>null</code>
+     */
+    public static final Property<String> LAST_SETUP_COMPLETED_HOST = new Property<>("last.setup.completed.host", null, String.class);
+
     /**
      * The time interval (in seconds) after which the agent will check if the connected host is the preferred host.<br>
      * After that interval, if the agent is connected to one of the secondary/backup hosts, it will attempt to reconnect to the preferred host.<br>

agent/src/test/java/com/cloud/agent/AgentShellTest.java

@@ -369,4 +369,49 @@ public void testGetSslHandshakeTimeout() {
         agentPropertiesFileHandlerMocked.when(() -> AgentPropertiesFileHandler.getPropertyValue(Mockito.eq(AgentProperties.SSL_HANDSHAKE_TIMEOUT))).thenReturn(expected);
         Assert.assertEquals(expected, agentShellSpy.getSslHandshakeTimeout());
     }
+
+    private void mockLastSetupCompletedHost(String value) {
+        agentPropertiesFileHandlerMocked.when(() -> AgentPropertiesFileHandler.getPropertyValue(Mockito.eq(AgentProperties.LAST_SETUP_COMPLETED_HOST))).thenReturn(value);
+    }
+
+    @Test
+    public void getHostsTestAppendsLastSetupCompletedHostAsFallback() {
+        mockLastSetupCompletedHost("30.3.3.3");
+        agentShellSpy.setHosts("10.1.1.1,20.2.2.2");
+
+        Assert.assertArrayEquals(new String[] {"10.1.1.1", "20.2.2.2", "30.3.3.3"}, agentShellSpy.getHosts());
+    }
+
+    @Test
+    public void getHostsTestSubstringHostDoesNotSuppressFallback() {
+        // 10.0.0.1 is a substring of 10.0.0.10 but not the same host, so it must still be appended.
+        mockLastSetupCompletedHost("10.0.0.1");
+        agentShellSpy.setHosts("10.0.0.10");
+
+        Assert.assertArrayEquals(new String[] {"10.0.0.10", "10.0.0.1"}, agentShellSpy.getHosts());
+    }
+
+    @Test
+    public void getHostsTestExactMatchIsNotDuplicated() {
+        mockLastSetupCompletedHost("20.2.2.2");
+        agentShellSpy.setHosts("10.1.1.1,20.2.2.2");
+
+        Assert.assertArrayEquals(new String[] {"10.1.1.1", "20.2.2.2"}, agentShellSpy.getHosts());
+    }
+
+    @Test
+    public void getHostsTestMatchIsCaseInsensitiveAndTrimmed() {
+        mockLastSetupCompletedHost("  HOSTA.EXAMPLE.COM  ");
+        agentShellSpy.setHosts("hosta.example.com,hostb.example.com");
+
+        Assert.assertArrayEquals(new String[] {"hosta.example.com", "hostb.example.com"}, agentShellSpy.getHosts());
+    }
+
+    @Test
+    public void getHostsTestBlankLastSetupCompletedHostReturnsConfiguredHostsOnly() {
+        mockLastSetupCompletedHost("  ");
+        agentShellSpy.setHosts("10.1.1.1,20.2.2.2");
+
+        Assert.assertArrayEquals(new String[] {"10.1.1.1", "20.2.2.2"}, agentShellSpy.getHosts());
+    }
 }

agent/src/test/java/com/cloud/agent/AgentTest.java

@@ -36,6 +36,7 @@
 import javax.naming.ConfigurationException;
 
 import org.apache.logging.log4j.Logger;
+import com.cloud.utils.nio.NioClient;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -45,7 +46,6 @@
 import com.cloud.resource.ServerResource;
 import com.cloud.utils.backoff.impl.ConstantTimeBackoff;
 import com.cloud.utils.nio.Link;
-import com.cloud.utils.nio.NioConnection;
 
 @RunWith(MockitoJUnitRunner.class)
 public class AgentTest {
@@ -224,7 +224,7 @@ public void testStopAndCleanupConnectionConnectionIsNullDoesNothing() {
 
     @Test
     public void testStopAndCleanupConnectionValidConnectionNoWaitStopsAndCleansUp() throws IOException {
-        NioConnection mockConnection = mock(NioConnection.class);
+        NioClient mockConnection = mock(NioClient.class);
         agent.connection = mockConnection;
         agent.stopAndCleanupConnection(false);
         verify(mockConnection).stop();
@@ -233,7 +233,7 @@ public void testStopAndCleanupConnectionValidConnectionNoWaitStopsAndCleansUp()
 
     @Test
     public void testStopAndCleanupConnectionCleanupThrowsIOExceptionLogsWarning() throws IOException {
-        NioConnection mockConnection = mock(NioConnection.class);
+        NioClient mockConnection = mock(NioClient.class);
         agent.connection = mockConnection;
         doThrow(new IOException("Cleanup failed")).when(mockConnection).cleanUp();
         agent.stopAndCleanupConnection(false);
@@ -243,7 +243,7 @@ public void testStopAndCleanupConnectionCleanupThrowsIOExceptionLogsWarning() th
 
     @Test
     public void testStopAndCleanupConnectionValidConnectionWaitForStopWaitsForStartupToStop() throws IOException {
-        NioConnection mockConnection = mock(NioConnection.class);
+        NioClient mockConnection = mock(NioClient.class);
         ConstantTimeBackoff mockBackoff = mock(ConstantTimeBackoff.class);
         mockBackoff.setTimeToWait(0);
         agent.connection = mockConnection;