Teamcity -server process went down

[dsrivastv]

Hello Plugin Team ,

Yesterday we faced an issue due to crowd connectivity , teamcity server went down . Detail logs given . Could you please help us here . How can we avoid Teamcity -server to be down in case if crowd connectivity is not there ?.

[2020-02-24 10:16:36,092] ERROR - jetbrains.buildServer.SERVER - Bummer. Something went wrong. Can't talk to Crowd at all.
com.atlassian.crowd.exception.OperationFailedException: org.apache.http.conn.ConnectTimeoutException: Connect to loadbalancer.server.net:443 [loadbalancer.server.net/xx.xxx.xxx.xxx (ip)] timed out
at com.atlassian.crowd.integration.rest.service.RestExecutor$MethodExecutor.andReceive(RestExecutor.java:373)
at com.atlassian.crowd.integration.rest.service.RestCrowdClient.authenticateUser(RestCrowdClient.java:144)
at teamcity.crowd.plugin.TeamCityPluginCrowdClient.loginUserWithPassword(PluginCrowdClient.kt:40)
at teamcity.crowd.plugin.CrowdLoginModule.login(CrowdLoginModule.kt:47)
at sun.reflect.GeneratedMethodAccessor5794.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at javax.security.auth.login.LoginContext.invoke(LoginContext.java:755)
at javax.security.auth.login.LoginContext.access$000(LoginContext.java:195)
at javax.security.auth.login.LoginContext$4.run(LoginContext.java:682)
at javax.security.auth.login.LoginContext$4.run(LoginContext.java:680)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:680)
at javax.security.auth.login.LoginContext.login(LoginContext.java:587)
at jetbrains.buildServer.serverSide.impl.auth.ServerLoginModelImpl.lambda$null$1(ServerLoginModelImpl.java:123)
at jetbrains.buildServer.plugins.PluginSupportUtil.switchClassLoader(PluginSupportUtil.java:19)
at jetbrains.buildServer.serverSide.impl.auth.ServerLoginModelImpl.lambda$doTryToLogin$2(ServerLoginModelImpl.java:103)
at jetbrains.buildServer.serverSide.impl.BaseAccessChecker.runWithDisabledCheck(BaseAccessChecker.java:8)
at jetbrains.buildServer.serverSide.impl.SecondaryNodeSecurityManager.runSafeNetworkOperation(SecondaryNodeSecurityManager.java:19)
at jetbrains.buildServer.serverSide.ReadOnlyRestrictor.doReadOnlyNetworkOperation(ReadOnlyRestrictor.java:5)
at jetbrains.buildServer.serverSide.impl.auth.ServerLoginModelImpl.doTryToLogin(ServerLoginModelImpl.java:120)
at jetbrains.buildServer.serverSide.impl.auth.ServerLoginModelImpl.lambda$tryToLogin$0(ServerLoginModelImpl.java:13)
at jetbrains.buildServer.util.NamedThreadFactory.executeWithNewThreadNameFuncThrow(NamedThreadFactory.java:107)
at jetbrains.buildServer.serverSide.impl.auth.ServerLoginModelImpl.tryToLogin(ServerLoginModelImpl.java:89)
at jetbrains.buildServer.serverSide.impl.auth.ServerLoginModelImpl.doCheckCredentials(ServerLoginModelImpl.java:168)
at jetbrains.buildServer.serverSide.impl.auth.ServerLoginModelImpl.checkCredentials(ServerLoginModelImpl.java:114)
at jetbrains.buildServer.controllers.interceptors.auth.impl.BasicHttpAuthenticationScheme.doUserAuthentication(BasicHttpAuthenticationScheme.java:2onHeaderBasedHttpAuthenticationScheme.java:5)
at jetbrains.buildServer.controllers.interceptors.auth.impl.HttpAuthenticationManagerImpl.lambda$doProcessAuthenticationRequest$0(HttpAuthenticationManagerImpl.java:77)
at jetbrains.buildServer.serverSide.impl.BaseAccessChecker.runWithDisabledCheck(BaseAccessChecker.java:8)
at jetbrains.buildServer.serverSide.impl.SecondaryNodeSecurityManager.runSafeNetworkOperation(SecondaryNodeSecurityManager.java:19)
at jetbrains.buildServer.serverSide.ReadOnlyRestrictor.doReadOnlyNetworkOperation(ReadOnlyRestrictor.java:5)
at jetbrains.buildServer.controllers.interceptors.auth.impl.HttpAuthenticationManagerImpl.doProcessAuthenticationRequest(HttpAuthenticationManagerImpl.java:51)
at jetbrains.buildServer.controllers.interceptors.auth.impl.HttpAuthenticationManagerImpl.processAuthenticationRequest(HttpAuthenticationManagerImpl.java:25)
at jetbrains.buildServer.controllers.interceptors.AuthorizationInterceptorImpl$1.call(AuthorizationInterceptorImpl.java:19)
at jetbrains.buildServer.controllers.interceptors.AuthorizationInterceptorImpl$1.call(AuthorizationInterceptorImpl.java:25)
at jetbrains.buildServer.util.NamedThreadFactory.executeWithNewThreadName(NamedThreadFactory.java:91)
at jetbrains.buildServer.controllers.interceptors.AuthorizationInterceptorImpl.preHandle(AuthorizationInterceptorImpl.java:34)
at jetbrains.buildServer.controllers.interceptors.RequestInterceptors.preHandle(RequestInterceptors.java:28)
at org.springframework.web.servlet.HandlerExecutionChain.applyPreHandle(HandlerExecutionChain.java:133)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:962)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:861)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:635)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:742)
at jetbrains.buildServer.maintenance.TeamCityDispatcherServlet.processedByMainServlet(TeamCityDispatcherServlet.java:25)
at jetbrains.buildServer.maintenance.TeamCityDispatcherServlet.service(TeamCityDispatcherServlet.java:28)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at jetbrains.buildServer.web.jsp.JspPrecompilerFilter.doFilter(JspPrecompilerFilter.java:57)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at jetbrains.buildServer.web.DisableSessionIdFromUrlFilter.doFilter(DisableSessionIdFromUrlFilter.java:2)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at jetbrains.buildServer.web.UserIdProviderFilter.doFilter(UserIdProviderFilter.java:4)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at jetbrains.buildServer.web.NodeInfoHeaderFilter.doFilter(NodeInfoHeaderFilter.java:9)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
at org.springframework.web.filter.CompositeFilter$VirtualFilterChain.doFilter(CompositeFilter.java:107)
at jetbrains.buildServer.diagnostic.web.DiagnosticFilter.doFilter(DiagnosticFilter.java:2)
at org.springframework.web.filter.CompositeFilter$VirtualFilterChain.doFilter(CompositeFilter.java:112)
at jetbrains.buildServer.web.DependencyParametersCalculationContextFilter.doFilter(DependencyParametersCalculationContextFilter.java:4)
at org.springframework.web.filter.CompositeFilter$VirtualFilterChain.doFilter(CompositeFilter.java:112)
at jetbrains.buildServer.web.HttpSecurityHeadersFilter.doFilter(HttpSecurityHeadersFilter.java:41)
at org.springframework.web.filter.CompositeFilter$VirtualFilterChain.doFilter(CompositeFilter.java:112)
at jetbrains.buildServer.web.CSRFFilter.doFilter(CSRFFilter.java:112)
at org.springframework.web.filter.CompositeFilter$VirtualFilterChain.doFilter(CompositeFilter.java:112)
at org.springframework.web.filter.CompositeFilter.doFilter(CompositeFilter.java:73)
at jetbrains.buildServer.web.DelegatingFilter.doFilter(DelegatingFilter.java:61)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)

Caused by: org.apache.http.conn.ConnectTimeoutException: Connect to loadbalancer.server.net:443 [loadbalancer.server.net/xx.xxx.xx.xxx (ip)] timed out
at org.apache.http.impl.conn.HttpClientConnectionOperator.connect(HttpClientConnectionOperator.java:134)
at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:319)
at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:363)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:219)
at org.apache.http.impl.client.cache.CachingExec.callBackend(CachingExec.java:593)
at org.apache.http.impl.client.cache.CachingExec.execute(CachingExec.java:270)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:195)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:86)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
at com.atlassian.crowd.integration.rest.service.RestExecutor$MethodExecutor.executeCrowdServiceMethod(RestExecutor.java:490)
at com.atlassian.crowd.integration.rest.service.RestExecutor$MethodExecutor.andReceive(RestExecutor.java:364)
... 98 more
Caused by: java.net.SocketTimeoutException
at java.net.SocksSocketImpl.remainingMillis(SocksSocketImpl.java:111)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:589)
at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:244)
at org.apache.http.impl.conn.HttpClientConnectionOperator.connect(HttpClientConnectionOperator.java:125)
... 110 more

[greggigon]

Hi @dsrivastv ,
In theory it should be possible but what would be the fallback mechanism?
If you can't Authorize/Authenticate user as the underlying security mechanism is dead, how do you ensure security?