Skip to content

Commit c000848

Browse files
authored
Merge branch 'trunk' into feature/signed-uploads-on-azure-and-s3
2 parents 1171660 + e5f6549 commit c000848

22 files changed

Lines changed: 444 additions & 110 deletions

CHANGES.rst

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,49 @@
11
Changelog
22
=========
33

4+
Changes in Apache Libcloud 3.9.1
5+
--------------------------------
6+
7+
Compute
8+
~~~~~~~
9+
10+
- [OpenStack] Initial Blazar support
11+
12+
This is an initial implementation of Blazar support in Libcloud. It currently
13+
supports listing the available leases and hosts.
14+
(#2094)
15+
[Miguel Caballer - @micafer]
16+
17+
18+
- [Azure ARM] Update US GovCloud AD endpoint for AZURE_ARM provider.
19+
(#2062)
20+
[Chris Clifford - @CrCliff]
21+
22+
- [OpenStack] Add hypervisor_hostname attribute to OpenStack node.
23+
(#2063)
24+
[Miguel Caballer - @micafer]
25+
26+
27+
- [GCP] Use the fully-qualified name for the GCP IMDS endpoint.
28+
(#2060)
29+
[Frederic Hemery - @Arkelenia]
30+
31+
DNS
32+
~~~~
33+
34+
- [Cloudflare] Removal of zone_name from Cloudflare record response following API deprecation
35+
36+
This change will cause that record name returned by the Cloudflare driver to be fully
37+
qualified (e.g. "www.example.com") instead of relative to the zone (e.g. "www").
38+
This is due to the fact that Cloudflare has deprecated the "zone_name" field in their
39+
API response and it is no longer available.
40+
(#2068)
41+
[Chris Smith - cha0tic87]
42+
43+
- [Luadns] Fix example keyword arguments.
44+
(#2064)
45+
[Aaron Sierra - @aaron-sierra]
46+
447
Changes in Apache Libcloud 3.9.0
548
--------------------------------
649

dist/verify_checksums.sh

Lines changed: 33 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,34 @@ TMP_DIR=$(mktemp -d)
3030
# TODO: Use json endpoint + jq to parse out the url
3131
# https://pypi.org/pypi/apache-libcloud/3.4.0/json
3232
EXTENSIONS[0]=".tar.gz"
33-
EXTENSIONS[1]="-py2.py3-none-any.whl"
33+
EXTENSIONS[1]=".whl"
3434

35-
APACHE_MIRROR_URL="http://www.apache.org/dist/libcloud"
36-
PYPI_MIRROR_URL_SOURCE="https://pypi.python.org/packages/source/a/apache-libcloud"
37-
PYPI_SIMPLE_URL="https://pypi.org/simple/apache-libcloud/"
35+
# Get the download URL for the given extension from PyPi JSON API
36+
function get_pypi_url() {
37+
local extension=$1
38+
local pypi_version
39+
40+
pypi_version=$(echo "${VERSION}" | sed -E "s/^apache[-_]libcloud-//")
41+
42+
curl -s "https://pypi.org/pypi/apache-libcloud/${pypi_version}/json" | \
43+
jq -r --arg ext "${extension}" \
44+
'.urls[] | select(.filename | endswith($ext)) | .url' | \
45+
head -n 1
46+
}
47+
48+
# Get the download URL for the given extension from Apache mirror
49+
function get_apache_url() {
50+
local extension=$1
51+
local apache_version
52+
53+
apache_version=$(echo "${VERSION}" | sed -E "s/^apache[-_]libcloud-//")
54+
55+
# List files from Apache directory and find the matching file
56+
curl -s "https://downloads.apache.org/libcloud/" | \
57+
grep -oP "href=\"\K[^\"]*-${apache_version}[^\"]*${extension}" | \
58+
head -n 1 | \
59+
sed "s|^|https://downloads.apache.org/libcloud/|"
60+
}
3861

3962
# From http://tldp.org/LDP/abs/html/debugging.html#ASSERT
4063
function assert () # If condition false,
@@ -69,19 +92,17 @@ do
6992
extension=${EXTENSIONS[$i]}
7093
file_name="${VERSION}${extension}"
7194

72-
if [ "${extension}" = "-py2.py3-none-any.whl" ]; then
95+
if [ "${extension}" = ".whl" ]; then
7396
# shellcheck disable=SC2001
7497
file_name=$(echo "${file_name}" | sed "s/apache-libcloud/apache_libcloud/g")
7598
fi
7699

77-
apache_url="${APACHE_MIRROR_URL}/${file_name}"
78-
pypi_url="${PYPI_MIRROR_URL}/${file_name}"
100+
apache_url=$(get_apache_url "${extension}")
101+
pypi_url=$(get_pypi_url "${extension}")
79102

80-
if [ "${extension}" = "-py2.py3-none-any.whl" ]; then
81-
# Get the wheel full URL from PyPi Simple index
82-
pypi_url=$(curl -s ${PYPI_SIMPLE_URL} | grep "${file_name}" | sed -n 's/.*href="\([^"]*\)".*/\1/p')
83-
else
84-
pypi_url="${PYPI_MIRROR_URL_SOURCE}/${file_name}"
103+
if [ -z "${pypi_url}" ]; then
104+
echo "[ERR] Failed to resolve PyPi URL for ${file_name}"
105+
exit 2
85106
fi
86107

87108
assert "${apache_url} != ${pypi_url}", "URLs must be different"

docs/examples/dns/luadns/instantiate_driver.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@
22
from libcloud.dns.providers import get_driver
33

44
cls = get_driver(Provider.LUADNS)
5-
driver = cls(user="user", key="api_key")
5+
driver = cls(key="user", secret="api_key")

libcloud/common/azure_arm.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ def parse_error(self):
101101
"sqlManagementEndpointUrl": "https://management.core.usgovcloudapi.net:8443/",
102102
"sqlServerHostnameSuffix": ".database.usgovcloudapi.net",
103103
"galleryEndpointUrl": "https://gallery.usgovcloudapi.net/",
104-
"activeDirectoryEndpointUrl": "https://login-us.microsoftonline.com",
104+
"activeDirectoryEndpointUrl": "https://login.microsoftonline.us",
105105
"activeDirectoryResourceId": "https://management.core.usgovcloudapi.net/",
106106
"activeDirectoryGraphResourceId": "https://graph.windows.net/",
107107
"activeDirectoryGraphApiVersion": "2013-04-05",

libcloud/common/google.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ def _from_utc_timestamp(timestamp):
126126

127127
def _get_gce_metadata(path="", retry_failed: Optional[bool] = None):
128128
try:
129-
url = "http://metadata/computeMetadata/v1/" + path.lstrip("/")
129+
url = "http://metadata.google.internal/computeMetadata/v1/" + path.lstrip("/")
130130
headers = {"Metadata-Flavor": "Google"}
131131
response = get_response_object(url, headers=headers, retry_failed=retry_failed)
132132
return response.status, "", response.body

libcloud/compute/drivers/openstack.py

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,12 @@ class OpenStackVolumeV3Connection(OpenStackBaseConnection):
114114
service_region = "RegionOne"
115115

116116

117+
class OpenStackReservationConnection(OpenStackBaseConnection):
118+
service_type = "reservation"
119+
service_name = "blazar"
120+
service_region = "RegionOne"
121+
122+
117123
class OpenStackNodeDriver(NodeDriver, OpenStackDriverMixin):
118124
"""
119125
Base OpenStack node driver. Should not be used directly.
@@ -2467,6 +2473,7 @@ def _to_node(self, api_node):
24672473
power_state=api_node.get("OS-EXT-STS:power_state", None),
24682474
progress=api_node.get("progress", None),
24692475
fault=api_node.get("fault"),
2476+
hypervisor_hostname=api_node.get("OS-EXT-SRV-ATTR:hypervisor_hostname"),
24702477
),
24712478
)
24722479

@@ -2813,6 +2820,15 @@ def encode_data(self, data):
28132820
return json.dumps(data)
28142821

28152822

2823+
class OpenStack_2_ReservationConnection(OpenStackReservationConnection):
2824+
responseCls = OpenStack_1_1_Response
2825+
accept_format = "application/json"
2826+
default_content_type = "application/json; charset=UTF-8"
2827+
2828+
def encode_data(self, data):
2829+
return json.dumps(data)
2830+
2831+
28162832
class OpenStack_2_PortInterfaceState(Type):
28172833
"""
28182834
Standard states of OpenStack_2_PortInterfaceState
@@ -2873,6 +2889,10 @@ class OpenStack_2_NodeDriver(OpenStack_1_1_NodeDriver):
28732889
volumev3_connection = None
28742890
volume_connection = None
28752891

2892+
# Connection to the Blazar reservation API
2893+
reservation_connectionCls = OpenStack_2_ReservationConnection
2894+
reservation_connection = None
2895+
28762896
type = Provider.OPENSTACK
28772897

28782898
features = {"create_node": ["generates_password"]}
@@ -2929,6 +2949,16 @@ def __init__(self, *args, **kwargs):
29292949
super().__init__(*args, **kwargs)
29302950
self.network_connection = self.connection
29312951

2952+
# We run the init once to get the Blazar API connection
2953+
# and put that on the object under self.reservation_connection.
2954+
if original_ex_force_base_url or kwargs.get("ex_force_reservation_url"):
2955+
kwargs["ex_force_base_url"] = str(
2956+
kwargs.pop("ex_force_reservation_url", original_ex_force_base_url)
2957+
)
2958+
self.connectionCls = self.reservation_connectionCls
2959+
super().__init__(*args, **kwargs)
2960+
self.reservation_connection = self.connection
2961+
29322962
# We run the init once again to get the compute API connection
29332963
# and that's put under self.connection as normal.
29342964
self._ex_force_base_url = original_ex_force_base_url
@@ -4368,6 +4398,100 @@ def ex_detach_floating_ip_from_node(self, node, ip):
43684398
)
43694399
return resp.status == httplib.OK
43704400

4401+
def ex_list_leases(self):
4402+
"""
4403+
List leases
4404+
4405+
:rtype: ``list`` of :class:`OpenStack_2_Lease`
4406+
"""
4407+
return self._to_leases(self.reservation_connection.request("/leases").object)
4408+
4409+
def _to_leases(self, obj):
4410+
lease_elements = obj["leases"]
4411+
return [self._to_lease(lease) for lease in lease_elements]
4412+
4413+
def _to_lease(self, obj):
4414+
return OpenStack_2_Lease(
4415+
id=obj["id"],
4416+
name=obj["name"],
4417+
start=obj["start_date"],
4418+
end=obj["end_date"],
4419+
status=obj["status"],
4420+
reservations=obj["reservations"],
4421+
driver=self.reservation_connection.driver,
4422+
)
4423+
4424+
def ex_list_hosts(self):
4425+
"""
4426+
List leases
4427+
4428+
:rtype: ``list`` of :class:`OpenStack_2_Host`
4429+
"""
4430+
return self._to_hosts(self.reservation_connection.request("/os-hosts").object)
4431+
4432+
def _to_hosts(self, obj):
4433+
host_elements = obj["hosts"]
4434+
return [self._to_host(host) for host in host_elements]
4435+
4436+
def _to_host(self, obj):
4437+
return OpenStack_2_Host(
4438+
id=obj["id"],
4439+
hypervisor_hostname=obj["hypervisor_hostname"],
4440+
vcpus=obj["vcpus"],
4441+
memory_mb=obj["memory_mb"],
4442+
local_gb=obj["local_gb"],
4443+
service_name=obj["service_name"],
4444+
)
4445+
4446+
4447+
class OpenStack_2_Host:
4448+
"""
4449+
Host info.
4450+
"""
4451+
4452+
def __init__(self, id, hypervisor_hostname, vcpus, memory_mb, local_gb, service_name):
4453+
self.id = id
4454+
self.hypervisor_hostname = hypervisor_hostname
4455+
self.vcpus = vcpus
4456+
self.memory_mb = memory_mb
4457+
self.local_gb = local_gb
4458+
self.service_name = service_name
4459+
4460+
def __repr__(self):
4461+
return "<OpenStack_2_Host: id={}, hypervisor_hostname={}>".format(
4462+
self.id,
4463+
self.hypervisor_hostname,
4464+
)
4465+
4466+
4467+
class OpenStack_2_Lease:
4468+
"""
4469+
Lease info.
4470+
"""
4471+
4472+
PENDING = "PENDING"
4473+
ACTIVE = "ACTIVE"
4474+
TERMINATED = "TERMINATED"
4475+
DELETED = "DELETED"
4476+
CREATING = "CREATING"
4477+
UPDATING = "UPDATING"
4478+
DELETING = "DELETING"
4479+
ERROR = "ERROR"
4480+
4481+
def __init__(self, id, name, start, end, status, reservations, driver):
4482+
self.id = id
4483+
self.name = name
4484+
self.start = start
4485+
self.end = end
4486+
self.status = status
4487+
self.reservations = reservations
4488+
self.driver = driver
4489+
4490+
def __repr__(self):
4491+
return "<OpenStack_2_Lease: id={}, name={}, status={}>".format(
4492+
self.id, self.name, self.status
4493+
)
4494+
43714495

43724496
class OpenStack_1_1_FloatingIpPool:
43734497
"""

libcloud/dns/drivers/cloudflare.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -516,8 +516,6 @@ def _to_zone(self, item):
516516

517517
def _to_record(self, zone, item):
518518
name = item["name"]
519-
name = name.replace("." + item["zone_name"], "")
520-
name = name.replace(item["zone_name"], "")
521519
name = name or None
522520

523521
ttl = item.get("ttl")

libcloud/test/common/test_openstack_identity.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -833,7 +833,7 @@ def test_parsing_auth_v2(self):
833833

834834
catalog = OpenStackServiceCatalog(service_catalog=service_catalog, auth_version="2.0")
835835
entries = catalog.get_entries()
836-
self.assertEqual(len(entries), 10)
836+
self.assertEqual(len(entries), 11)
837837

838838
entry = [e for e in entries if e.service_name == "cloudServers"][0]
839839
self.assertEqual(entry.service_type, "compute")
@@ -904,6 +904,7 @@ def test_get_service_types(self):
904904
"network",
905905
"object-store",
906906
"rax:object-cdn",
907+
"reservation",
907908
"volumev2",
908909
"volumev3",
909910
],
@@ -923,6 +924,7 @@ def test_get_service_names(self):
923924
self.assertEqual(
924925
service_names,
925926
[
927+
"blazar",
926928
"cinderv2",
927929
"cinderv3",
928930
"cloudFiles",

libcloud/test/compute/fixtures/gce/project.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
},
1616
{
1717
"key": "startup-script",
18-
"value": "#!/bin/bash\n\nAUTO_SCRIPT=$(curl -s http://metadata/computeMetadata/v1/instance/attributes/my-auto-script -H \"Metadata-Flavor: Google\")\nCHECK=${AUTO_SCRIPT:-disabled}\n\nif [ \"${CHECK}\" = \"enabled\" -a -f /etc/debian_version ]; then\n export DEBIAN_FRONTEND=noninteractive\n apt-get -q -y update\n apt-get -q -y install git vim tmux\n fi\nexit 0\n"
18+
"value": "#!/bin/bash\n\nAUTO_SCRIPT=$(curl -s http://metadata.google.internal/computeMetadata/v1/instance/attributes/my-auto-script -H \"Metadata-Flavor: Google\")\nCHECK=${AUTO_SCRIPT:-disabled}\n\nif [ \"${CHECK}\" = \"enabled\" -a -f /etc/debian_version ]; then\n export DEBIAN_FRONTEND=noninteractive\n apt-get -q -y update\n apt-get -q -y install git vim tmux\n fi\nexit 0\n"
1919
}
2020
]
2121
},

libcloud/test/compute/fixtures/openstack/_v2_0__auth.json

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,20 @@
164164
"name": "cinderv3",
165165
"type": "volumev3"
166166
},
167+
{
168+
"endpoints": [
169+
{
170+
"region": "RegionOne",
171+
"tenantId": "1337",
172+
"publicURL": "https://test_endpoint.com/v1",
173+
"versionInfo": "https://test_endpoint.com/v1/",
174+
"versionList": "https://test_endpoint.com/",
175+
"versionId": "1"
176+
}
177+
],
178+
"name": "blazar",
179+
"type": "reservation"
180+
},
167181
{
168182
"endpoints": [
169183
{

0 commit comments

Comments
 (0)