diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 77fa6a0f35544a..ce48d7ef189f3c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -1406,8 +1406,7 @@ protected void getClusterIdAndRole() throws IOException { getClusterIdFromStorage(storage); token = storage.getToken(); try { - String url = "http://" + NetUtils - .getHostPortInAccessibleFormat(rightHelperNode.getHost(), Config.http_port) + "/check"; + String url = HttpURLUtil.buildInternalFeUrl(rightHelperNode.getHost(), "/check", null); HttpURLConnection conn = HttpURLUtil.getConnectionWithNodeIdent(url); conn.setConnectTimeout(2 * 1000); conn.setReadTimeout(2 * 1000); @@ -1503,9 +1502,8 @@ protected boolean getFeNodeTypeAndNameFromHelpers() { try { // For upgrade compatibility, the host parameter name remains the same // and the new hostname parameter is added - String url = "http://" + NetUtils.getHostPortInAccessibleFormat(helperNode.getHost(), Config.http_port) - + "/role?host=" + selfNode.getHost() - + "&port=" + selfNode.getPort(); + String queryParams = "host=" + selfNode.getHost() + "&port=" + selfNode.getPort(); + String url = HttpURLUtil.buildInternalFeUrl(helperNode.getHost(), "/role", queryParams); HttpURLConnection conn = HttpURLUtil.getConnectionWithNodeIdent(url); if (conn.getResponseCode() != 200) { LOG.warn("failed to get fe node type from helper node: {}. response code: {}", helperNode, @@ -1768,7 +1766,7 @@ private void transferToMaster() { toMasterProgress = "log master info"; this.masterInfo = new MasterInfo(Env.getCurrentEnv().getSelfNode().getHost(), - Config.http_port, + Config.enable_https ? Config.https_port : Config.http_port, Config.rpc_port); editLog.logMasterInfo(masterInfo); LOG.info("logMasterInfo:{}", masterInfo); @@ -2129,8 +2127,7 @@ private void checkBeExecVersion() { protected boolean getVersionFileFromHelper(HostInfo helperNode) throws IOException { try { - String url = "http://" + NetUtils.getHostPortInAccessibleFormat(helperNode.getHost(), Config.http_port) - + "/version"; + String url = HttpURLUtil.buildInternalFeUrl(helperNode.getHost(), "/version", null); File dir = new File(this.imageDir); MetaHelper.getRemoteFile(url, HTTP_TIMEOUT_SECOND * 1000, MetaHelper.getFile(Storage.VERSION_FILE, dir)); @@ -2149,8 +2146,7 @@ protected void getNewImage(HostInfo helperNode) throws IOException { localImageVersion = storage.getLatestImageSeq(); try { - String hostPort = NetUtils.getHostPortInAccessibleFormat(helperNode.getHost(), Config.http_port); - String infoUrl = "http://" + hostPort + "/info"; + String infoUrl = HttpURLUtil.buildInternalFeUrl(helperNode.getHost(), "/info", null); ResponseBody responseBody = MetaHelper .doGet(infoUrl, HTTP_TIMEOUT_SECOND * 1000, StorageInfo.class); if (responseBody.getCode() != RestApiStatusCode.OK.code) { @@ -2160,7 +2156,7 @@ protected void getNewImage(HostInfo helperNode) throws IOException { StorageInfo info = responseBody.getData(); long version = info.getImageSeq(); if (version > localImageVersion) { - String url = "http://" + hostPort + "/image?version=" + version; + String url = HttpURLUtil.buildInternalFeUrl(helperNode.getHost(), "/image", "version=" + version); String filename = Storage.IMAGE + "." + version; File dir = new File(this.imageDir); MetaHelper.getRemoteFile(url, Config.sync_image_timeout_second * 1000, diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/HttpURLUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/HttpURLUtil.java index 70b3e68450aa45..b5efb317cabaa0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/HttpURLUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/HttpURLUtil.java @@ -19,14 +19,17 @@ import org.apache.doris.catalog.Env; import org.apache.doris.cloud.security.SecurityChecker; +import org.apache.doris.common.Config; import org.apache.doris.system.SystemInfoService.HostInfo; import com.google.common.collect.Maps; +import org.apache.http.conn.ssl.NoopHostnameVerifier; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; import java.util.Map; +import javax.net.ssl.HttpsURLConnection; public class HttpURLUtil { @@ -35,6 +38,13 @@ public static HttpURLConnection getConnectionWithNodeIdent(String request) throw SecurityChecker.getInstance().startSSRFChecking(request); URL url = new URL(request); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + + if (conn instanceof HttpsURLConnection && Config.enable_https) { + HttpsURLConnection httpsConn = (HttpsURLConnection) conn; + httpsConn.setSSLSocketFactory(InternalHttpsUtils.getSslContext().getSocketFactory()); + httpsConn.setHostnameVerifier(NoopHostnameVerifier.INSTANCE); + } + // Must use Env.getServingEnv() instead of getCurrentEnv(), // because here we need to obtain selfNode through the official service catalog. HostInfo selfNode = Env.getServingEnv().getSelfNode(); @@ -58,4 +68,16 @@ public static Map getNodeIdentHeaders() throws IOException { return headers; } + public static String buildInternalFeUrl(String host, String path, String queryParams) { + String protocol = Config.enable_https ? "https" : "http"; + int port = Config.enable_https ? Config.https_port : Config.http_port; + + String url = protocol + "://" + NetUtils.getHostPortInAccessibleFormat(host, port) + path; + if (queryParams != null && !queryParams.isEmpty()) { + url += "?" + queryParams; + } + + return url; + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/InternalHttpsUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/InternalHttpsUtils.java new file mode 100644 index 00000000000000..999754dfc776ca --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/InternalHttpsUtils.java @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.util; + +import org.apache.doris.common.Config; + +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.KeyStore; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; + +/** + * Utility for creating SSL-aware HTTP clients for internal FE-to-FE communication. + * + *

Builds an {@link SSLContext} from the configured CA truststore once and caches it. + * Hostname verification is disabled for IP-based intra-cluster connections. + * Certificate rotation requires a FE restart. + */ +public class InternalHttpsUtils { + private static volatile SSLContext cachedSslContext = null; + + private static final Logger LOG = LogManager.getLogger(InternalHttpsUtils.class); + + /** + * Returns the cached SSLContext, building it from the configured truststore on first call. + */ + public static SSLContext getSslContext() { + if (cachedSslContext == null) { + synchronized (InternalHttpsUtils.class) { + if (cachedSslContext == null) { + cachedSslContext = buildSslContext(); + } + } + } + return cachedSslContext; + } + + private static SSLContext buildSslContext() { + try { + KeyStore trustStore = KeyStore.getInstance(Config.ssl_trust_store_type); + try (InputStream stream = Files.newInputStream( + Paths.get(Config.mysql_ssl_default_ca_certificate))) { + trustStore.load(stream, Config.mysql_ssl_default_ca_certificate_password.toCharArray()); + } + + TrustManagerFactory tmf = TrustManagerFactory.getInstance( + TrustManagerFactory.getDefaultAlgorithm()); + tmf.init(trustStore); + + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, tmf.getTrustManagers(), null); + return sslContext; + } catch (Exception e) { + LOG.error("Failed to build SSLContext from truststore: {}", + Config.mysql_ssl_default_ca_certificate, e); + throw new RuntimeException("Failed to build SSLContext from truststore", e); + } + } + + /** + * Returns an HTTP client configured with the FE CA truststore and hostname verification disabled. + */ + public static CloseableHttpClient createValidatedHttpClient() { + SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory( + getSslContext(), + NoopHostnameVerifier.INSTANCE); + + return HttpClients.custom() + .setSSLSocketFactory(sslFactory) + .build(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/WebServerFactoryCustomizerConfig.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/WebServerFactoryCustomizerConfig.java index fc24bf1bd27884..593ce76f666f1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/WebServerFactoryCustomizerConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/WebServerFactoryCustomizerConfig.java @@ -25,6 +25,7 @@ import org.eclipse.jetty.ee10.websocket.server.config.JettyWebSocketServletContainerInitializer; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.ServerConnector; import org.springframework.boot.web.embedded.jetty.ConfigurableJettyWebServerFactory; import org.springframework.boot.web.embedded.jetty.JettyServletWebServerFactory; @@ -53,12 +54,22 @@ public void customize(ConfigurableJettyWebServerFactory factory) { }); if (Config.enable_https) { - ((JettyServletWebServerFactory) factory).setConfigurations( + if (!(factory instanceof JettyServletWebServerFactory)) { + LOG.warn("Unexpected WebServerFactory type {}, skipping HTTPS configuration", + factory.getClass().getName()); + return; + } + JettyServletWebServerFactory jettyFactory = (JettyServletWebServerFactory) factory; + jettyFactory.setConfigurations( Collections.singletonList(new HttpToHttpsJettyConfig()) ); factory.addServerCustomizers(server -> { if (server.getConnectors() != null && server.getConnectors().length > 0) { + if (!(server.getConnectors()[0] instanceof ServerConnector)) { + LOG.warn("First connector is not a ServerConnector, cannot configure SNI"); + return; + } ServerConnector existingConnector = (ServerConnector) server.getConnectors()[0]; HttpConnectionFactory httpFactory = existingConnector.getConnectionFactory(HttpConnectionFactory.class); @@ -66,6 +77,18 @@ public void customize(ConfigurableJettyWebServerFactory factory) { HttpConfiguration httpConfig = httpFactory.getHttpConfiguration(); httpConfig.setSecurePort(Config.https_port); httpConfig.setSecureScheme("https"); + + // Disable SNI host checking to allow IP-based connections + // Safe because checkFromValidFe() validates only registered FE nodes can connect + SecureRequestCustomizer secureCustomizer = + httpConfig.getCustomizer(SecureRequestCustomizer.class); + if (secureCustomizer == null) { + secureCustomizer = new SecureRequestCustomizer(); + httpConfig.addCustomizer(secureCustomizer); + } + secureCustomizer.setSniHostCheck(false); + secureCustomizer.setSniRequired(false); + LOG.info("Disabled SNI host checking for IP-based HTTPS connections"); } } }); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/SessionController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/SessionController.java index 94be8e173765ce..a753305106a94b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/SessionController.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/SessionController.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.SchemaTable; import org.apache.doris.catalog.Table; +import org.apache.doris.common.util.HttpURLUtil; import org.apache.doris.httpv2.entity.ResponseBody; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; import org.apache.doris.httpv2.rest.RestBaseController; @@ -118,8 +119,8 @@ private List> getOtherSessionInfo(HttpServletRequest request Frontend frontend) throws IOException { Map header = Maps.newHashMap(); header.put(NodeAction.AUTHORIZATION, request.getHeader(NodeAction.AUTHORIZATION)); - String res = HttpUtils.doGet(String.format("http://%s:%s/rest/v1/session", - frontend.getHost(), Env.getCurrentEnv().getMasterHttpPort()), header); + String res = HttpUtils.doGet(HttpURLUtil.buildInternalFeUrl( + frontend.getHost(), "/rest/v1/session", null), header); ObjectMapper objectMapper = new ObjectMapper(); Map jsonMap = objectMapper.readValue(res, new TypeReference>() {}); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java index 6a8bd71facaa79..04968eda8bdbde 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/meta/MetaService.java @@ -20,7 +20,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; -import org.apache.doris.common.util.NetUtils; +import org.apache.doris.common.util.HttpURLUtil; import org.apache.doris.ha.FrontendNodeType; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; import org.apache.doris.httpv2.rest.RestBaseController; @@ -163,8 +163,7 @@ public Object put(HttpServletRequest request, HttpServletResponse response) thro clientHost, clientPort, machine, portStr); clientHost = Strings.isNullOrEmpty(clientHost) ? machine : clientHost; - String url = "http://" + NetUtils.getHostPortInAccessibleFormat(clientHost, Integer.valueOf(portStr)) - + "/image?version=" + versionStr; + String url = HttpURLUtil.buildInternalFeUrl(clientHost, "/image", "version=" + versionStr); String filename = Storage.IMAGE + "." + versionStr; File dir = new File(Env.getCurrentEnv().getImageDir()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/ClusterAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/ClusterAction.java index 995bc8080e885c..370b15ef73fc67 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/ClusterAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/ClusterAction.java @@ -72,7 +72,8 @@ public Object clusterInfo(HttpServletRequest request, HttpServletResponse respon result.put("mysql", frontends.stream().map(ip -> NetUtils .getHostPortInAccessibleFormat(ip, Config.query_port)).collect(Collectors.toList())); result.put("http", frontends.stream().map(ip -> NetUtils - .getHostPortInAccessibleFormat(ip, Config.http_port)).collect(Collectors.toList())); + .getHostPortInAccessibleFormat(ip, + Config.enable_https ? Config.https_port : Config.http_port)).collect(Collectors.toList())); result.put("arrow flight sql server", frontends.stream().map( ip -> NetUtils.getHostPortInAccessibleFormat(ip, Config.arrow_flight_sql_port)) .collect(Collectors.toList())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/HttpUtils.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/HttpUtils.java index 694e55729526b7..7b8629550b143a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/HttpUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/HttpUtils.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; import org.apache.doris.common.Pair; +import org.apache.doris.common.util.InternalHttpsUtils; import org.apache.doris.common.util.Util; import org.apache.doris.httpv2.entity.ResponseBody; import org.apache.doris.persist.gson.GsonUtils; @@ -51,7 +52,7 @@ import java.util.stream.Collectors; /* - * used to forward http requests from manager to be. + * Used for internal HTTP(S) communication between FE nodes and from manager to BE. */ public class HttpUtils { private static final Logger LOG = LogManager.getLogger(HttpUtils.class); @@ -60,8 +61,9 @@ public class HttpUtils { static final int DEFAULT_TIME_OUT_MS = 2000; static List> getFeList() { + int port = Config.enable_https ? Config.https_port : Config.http_port; return Env.getCurrentEnv().getFrontends(null) - .stream().filter(Frontend::isAlive).map(fe -> Pair.of(fe.getHost(), Config.http_port)) + .stream().filter(Frontend::isAlive).map(fe -> Pair.of(fe.getHost(), port)) .collect(Collectors.toList()); } @@ -71,7 +73,7 @@ static boolean isCurrentFe(String ip, int port) { } static String concatUrl(Pair ipPort, String path, Map arguments) { - StringBuilder url = new StringBuilder("http://") + StringBuilder url = new StringBuilder(Config.enable_https ? "https://" : "http://") .append(ipPort.first).append(":").append(ipPort.second).append(path); boolean isFirst = true; for (Map.Entry entry : arguments.entrySet()) { @@ -130,8 +132,11 @@ public static CloseableHttpClient getHttpClient() { } private static String executeRequest(HttpRequestBase request) throws IOException { - CloseableHttpClient client = getHttpClient(); - return client.execute(request, httpResponse -> EntityUtils.toString(httpResponse.getEntity())); + try (CloseableHttpClient client = Config.enable_https + ? InternalHttpsUtils.createValidatedHttpClient() + : HttpClientBuilder.create().build()) { + return client.execute(request, httpResponse -> EntityUtils.toString(httpResponse.getEntity())); + } } static String parseResponse(String response) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java index 7e24078934570f..22d83c1d2addcb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java @@ -239,8 +239,9 @@ public Object nodeList(HttpServletRequest request, HttpServletResponse response) } private static List getFeList() { + int port = Config.enable_https ? Config.https_port : Config.http_port; return Env.getCurrentEnv().getFrontends(null).stream() - .map(fe -> NetUtils.getHostPortInAccessibleFormat(fe.getHost(), Config.http_port)) + .map(fe -> NetUtils.getHostPortInAccessibleFormat(fe.getHost(), port)) .collect(Collectors.toList()); } @@ -496,7 +497,8 @@ public Object setConfigFe(HttpServletRequest request, HttpServletResponse respon List> failedTotal = Lists.newArrayList(); List nodeConfigList = parseSetConfigNodes(requestBody, failedTotal); List> aliveFe = Env.getCurrentEnv().getFrontends(null).stream().filter(Frontend::isAlive) - .map(fe -> Pair.of(fe.getHost(), Config.http_port)).collect(Collectors.toList()); + .map(fe -> Pair.of(fe.getHost(), + Config.enable_https ? Config.https_port : Config.http_port)).collect(Collectors.toList()); checkNodeIsAlive(nodeConfigList, aliveFe, failedTotal); Map header = Maps.newHashMap(); @@ -568,7 +570,8 @@ private static void addFailedConfig(String configName, String value, String node private String concatFeSetConfigUrl(NodeConfigs nodeConfigs, boolean isPersist) { StringBuilder sb = new StringBuilder(); Pair hostPort = nodeConfigs.getHostPort(); - sb.append("http://").append(hostPort.first).append(":").append(hostPort.second).append("/api/_set_config"); + sb.append(Config.enable_https ? "https://" : "http://") + .append(hostPort.first).append(":").append(hostPort.second).append("/api/_set_config"); Map configs = nodeConfigs.getConfigs(isPersist); boolean addAnd = false; for (Map.Entry entry : configs.entrySet()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/QueryProfileAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/QueryProfileAction.java index 7af8a05b648d85..71ba5ef83982c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/QueryProfileAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/QueryProfileAction.java @@ -195,7 +195,7 @@ public Object queryInfo(HttpServletRequest request, HttpServletResponse response // add node information for (List query : queries) { query.set(1, NetUtils.getHostPortInAccessibleFormat(Env.getCurrentEnv().getSelfNode().getHost(), - Config.http_port)); + Config.enable_https ? Config.https_port : Config.http_port)); } if (!Strings.isNullOrEmpty(search)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java b/fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java index ec4126aa86d3b6..41386aae433743 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/Checkpoint.java @@ -23,7 +23,6 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.util.HttpURLUtil; import org.apache.doris.common.util.MasterDaemon; -import org.apache.doris.common.util.NetUtils; import org.apache.doris.httpv2.entity.ResponseBody; import org.apache.doris.httpv2.rest.RestApiStatusCode; import org.apache.doris.metric.MetricRepo; @@ -207,10 +206,10 @@ public synchronized void doCheckpoint() throws CheckpointException { // skip master itself continue; } - int port = Config.http_port; + int port = Config.enable_https ? Config.https_port : Config.http_port; - String url = "http://" + NetUtils.getHostPortInAccessibleFormat(host, port) + "/put?version=" + replayedJournalId - + "&port=" + port; + String queryParams = "version=" + replayedJournalId + "&port=" + port; + String url = HttpURLUtil.buildInternalFeUrl(host, "/put", queryParams); LOG.info("Put image:{}", url); try { @@ -264,7 +263,6 @@ private void deleteOldJournalsAndImages(int successPushed, int otherNodesCount, // skip master itself continue; } - int port = Config.http_port; String idURL; HttpURLConnection conn = null; try { @@ -274,7 +272,7 @@ private void deleteOldJournalsAndImages(int successPushed, int otherNodesCount, * any non-master node's current replayed journal id. otherwise, * this lagging node can never get the deleted journal. */ - idURL = "http://" + NetUtils.getHostPortInAccessibleFormat(host, port) + "/journal_id"; + idURL = HttpURLUtil.buildInternalFeUrl(host, "/journal_id", null); conn = HttpURLUtil.getConnectionWithNodeIdent(idURL); conn.setConnectTimeout(CONNECT_TIMEOUT_SECOND * 1000); conn.setReadTimeout(READ_TIMEOUT_SECOND * 1000); @@ -284,6 +282,7 @@ private void deleteOldJournalsAndImages(int successPushed, int otherNodesCount, minOtherNodesJournalId = id; } } catch (Throwable e) { + int port = Config.enable_https ? Config.https_port : Config.http_port; throw new CheckpointException(String.format("Exception when getting current replayed" + " journal id. host=%s, port=%d", host, port), e); } finally { diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/MetaHelper.java b/fe/fe-core/src/main/java/org/apache/doris/master/MetaHelper.java index 96e7375679150d..a2e61d2cb88fd3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/MetaHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/MetaHelper.java @@ -149,9 +149,13 @@ private static void checkFile(File file) throws IOException { public static ResponseBody doGet(String url, int timeout, Class clazz) throws IOException { Map headers = HttpURLUtil.getNodeIdentHeaders(); - LOG.info("meta helper, url: {}, timeout{}, headers: {}", url, timeout, headers); + LOG.debug("meta helper, url: {}, timeout: {}, headers: {}", url, timeout, headers); String response = HttpUtils.doGet(url, headers, timeout); - return parseResponse(response, clazz); + try { + return parseResponse(response, clazz); + } catch (Exception e) { + throw new IOException("Failed to parse response from " + url + ". Response: " + response, e); + } } // download file from remote node diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/HttpURLUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/HttpURLUtilTest.java new file mode 100644 index 00000000000000..5bba968b49af41 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/HttpURLUtilTest.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.util; + +import org.apache.doris.common.Config; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +public class HttpURLUtilTest { + + @After + public void tearDown() { + Config.enable_https = false; + Config.http_port = 8030; + Config.https_port = 8050; + } + + @Test + public void testBuildInternalFeUrlHttp() { + Config.enable_https = false; + Config.http_port = 8030; + + String url = HttpURLUtil.buildInternalFeUrl("192.168.1.10", "/put", "version=123&port=8030"); + Assert.assertEquals("http://192.168.1.10:8030/put?version=123&port=8030", url); + } + + @Test + public void testBuildInternalFeUrlHttps() { + Config.enable_https = true; + Config.https_port = 8050; + + String url = HttpURLUtil.buildInternalFeUrl("192.168.1.10", "/put", "version=123&port=8050"); + Assert.assertEquals("https://192.168.1.10:8050/put?version=123&port=8050", url); + } + + @Test + public void testBuildInternalFeUrlNoQueryParams() { + Config.enable_https = false; + Config.http_port = 8030; + + String url = HttpURLUtil.buildInternalFeUrl("192.168.1.10", "/journal_id", null); + Assert.assertEquals("http://192.168.1.10:8030/journal_id", url); + } + + @Test + public void testBuildInternalFeUrlEmptyQueryParams() { + Config.enable_https = false; + Config.http_port = 8030; + + String url = HttpURLUtil.buildInternalFeUrl("192.168.1.10", "/version", ""); + Assert.assertEquals("http://192.168.1.10:8030/version", url); + } + + @Test + public void testBuildInternalFeUrlHttpsWithIPv6() { + Config.enable_https = true; + Config.https_port = 8050; + + String url = HttpURLUtil.buildInternalFeUrl("fe80::1", "/role", "host=fe80::2&port=9010"); + Assert.assertTrue(url.startsWith("https://")); + Assert.assertTrue(url.contains("/role?host=fe80::2&port=9010")); + } +}