add user-agent header to template downloader request (#12791)

Author: sudo87

core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java

@@ -52,6 +52,7 @@
 import com.cloud.utils.Pair;
 import com.cloud.utils.UriUtils;
 import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.net.HttpClientCloudStackUserAgent;
 import com.cloud.utils.net.Proxy;
 
 /**
@@ -125,6 +126,7 @@ private GetMethod createRequest(String downloadUrl) {
         GetMethod request = new GetMethod(downloadUrl);
         request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
         request.setFollowRedirects(followRedirects);
+        request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT);
         return request;
     }
 

core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java

@@ -18,8 +18,11 @@
 //
 package com.cloud.storage.template;
 
+
 import com.cloud.storage.StorageLayer;
 import com.cloud.utils.UriUtils;
+import com.cloud.utils.net.HttpClientCloudStackUserAgent;
+
 import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.httpclient.HttpMethod;
 import org.apache.commons.httpclient.HttpMethodRetryHandler;
@@ -59,6 +62,7 @@ protected GetMethod createRequest(String downloadUrl) {
         GetMethod request = new GetMethod(downloadUrl);
         request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
         request.setFollowRedirects(followRedirects);
+        request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT);
         if (!toFileSet) {
             String[] parts = downloadUrl.split("/");
             String filename = parts[parts.length - 1];

core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java

@@ -44,6 +44,7 @@
 import org.apache.commons.lang3.StringUtils;
 
 import com.cloud.storage.StorageLayer;
+import com.cloud.utils.net.HttpClientCloudStackUserAgent;
 
 public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implements TemplateDownloader {
     private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
@@ -95,6 +96,7 @@ private GetMethod createRequest(String downloadUrl) {
         GetMethod request = new GetMethod(downloadUrl);
         request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler);
         request.setFollowRedirects(followRedirects);
+        request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT);
         return request;
     }
 
@@ -141,6 +143,7 @@ private void tryAndGetTotalRemoteSize() {
                 continue;
             }
             HeadMethod headMethod = new HeadMethod(downloadUrl);
+            headMethod.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT);
             try {
                 if (client.executeMethod(headMethod) != HttpStatus.SC_OK) {
                     continue;

core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java

@@ -19,6 +19,7 @@
 
 package org.apache.cloudstack.direct.download;
 
+
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -32,13 +33,15 @@
 import com.cloud.utils.Pair;
 import com.cloud.utils.UriUtils;
 import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.net.HttpClientCloudStackUserAgent;
 import com.cloud.utils.storage.QCOW2Utils;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.httpclient.HttpStatus;
 import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
 import org.apache.commons.httpclient.methods.GetMethod;
 import org.apache.commons.httpclient.methods.HeadMethod;
+import org.apache.commons.httpclient.params.HttpMethodParams;
 import org.apache.commons.io.IOUtils;
 
 public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
@@ -68,6 +71,7 @@ public HttpDirectTemplateDownloader(String url, Long templateId, String destPool
     protected GetMethod createRequest(String downloadUrl, Map<String, String> headers) {
         GetMethod request = new GetMethod(downloadUrl);
         request.setFollowRedirects(this.isFollowRedirects());
+        request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT);
         if (MapUtils.isNotEmpty(headers)) {
             for (String key : headers.keySet()) {
                 request.setRequestHeader(key, headers.get(key));
@@ -111,6 +115,7 @@ protected Pair<Boolean, String> performDownload() {
     public boolean checkUrl(String url) {
         HeadMethod httpHead = new HeadMethod(url);
         httpHead.setFollowRedirects(this.isFollowRedirects());
+        httpHead.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT);
         try {
             int responseCode = client.executeMethod(httpHead);
             if (responseCode != HttpStatus.SC_OK) {

utils/src/main/java/com/cloud/utils/HttpUtils.java

@@ -19,6 +19,8 @@
 
 package com.cloud.utils;
 
+import static com.cloud.utils.UriUtils.USER_AGENT;
+
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
 
@@ -33,6 +35,8 @@
 import java.net.URL;
 import java.util.Map;
 
+import com.cloud.utils.net.HttpClientCloudStackUserAgent;
+
 public class HttpUtils {
 
     protected static Logger LOGGER = LogManager.getLogger(HttpUtils.class);
@@ -161,6 +165,7 @@ public static boolean downloadFileWithProgress(final String fileURL, final Strin
         try {
             URL url = new URL(fileURL);
             httpConn = (HttpURLConnection) url.openConnection();
+            httpConn.setRequestProperty(USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT);
             int responseCode = httpConn.getResponseCode();
             if (responseCode == HttpURLConnection.HTTP_OK) {
                 int contentLength = httpConn.getContentLength();

utils/src/main/java/com/cloud/utils/UriUtils.java

@@ -67,12 +67,14 @@
 
 import com.cloud.utils.crypt.DBEncryptionUtil;
 import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.net.HttpClientCloudStackUserAgent;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 
 public class UriUtils {
 
     protected static Logger LOGGER = LogManager.getLogger(UriUtils.class);
+    public static final String USER_AGENT = "User-Agent";
 
     public static String formNfsUri(String host, String path) {
         try {
@@ -227,6 +229,7 @@ public static long getRemoteSize(String url, Boolean followRedirect) {
                 URI uri = new URI(url);
                 httpConn = (HttpURLConnection)uri.toURL().openConnection();
                 httpConn.setRequestMethod(method);
+                httpConn.setRequestProperty(USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT);
                 httpConn.setConnectTimeout(2000);
                 httpConn.setReadTimeout(5000);
                 httpConn.setInstanceFollowRedirects(Boolean.TRUE.equals(followRedirect));

utils/src/main/java/com/cloud/utils/net/HttpClientCloudStackUserAgent.java

@@ -0,0 +1,39 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.utils.net;
+
+import org.apache.logging.log4j.util.Strings;
+
+public class HttpClientCloudStackUserAgent {
+    public static final String CLOUDSTACK_USER_AGENT = buildUserAgent();
+
+    private static String buildUserAgent() {
+        String version = HttpClientCloudStackUserAgent.class
+                .getPackage()
+                .getImplementationVersion();
+
+        if (Strings.isBlank(version)) {
+            version = "unknown";
+        }
+        return "CloudStack-Agent/" + version + " (Apache CloudStack)";
+    }
+
+    private HttpClientCloudStackUserAgent() {}
+}

utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java

@@ -35,6 +35,7 @@
 
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.UriUtils;
+import com.cloud.utils.net.HttpClientCloudStackUserAgent;
 
 public final class QCOW2Utils {
     protected static Logger LOGGER = LogManager.getLogger(QCOW2Utils.class);
@@ -119,6 +120,7 @@ public static long getVirtualSizeFromUrl(String urlStr, boolean followRedirects)
         try {
             URI url = new URI(urlStr);
             httpConn = (HttpURLConnection)url.toURL().openConnection();
+            httpConn.setRequestProperty(UriUtils.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT);
             httpConn.setInstanceFollowRedirects(followRedirects);
             return getVirtualSize(httpConn.getInputStream(), UriUtils.isUrlForCompressedFile(urlStr));
         } catch (URISyntaxException e) {