Skip to content

Commit c008c00

Browse files
ykim-akamaiykim-akamai
authored
test: Add Calico Inbound and Outbound policies to LKE nodes for E2E (#619)
1 parent 8ea0a3d commit c008c00

3 files changed

Lines changed: 157 additions & 7 deletions

File tree

‎.github/workflows/e2e-suite.yml‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,19 @@ jobs:
2626
with:
2727
python-version: '3.x'
2828

29-
- name: Install Python deps
30-
run: pip install wheel boto3
31-
32-
- name: Update cert
33-
run: pip install certifi -U
29+
- name: Install Python dependencies and update cert
30+
run: |
31+
pip install wheel boto3 && \
32+
pip install certifi -U && \
33+
pip install .[obj,dev]
3434
35-
- name: Install deps
36-
run: pip install .[obj,dev]
35+
- name: Download kubectl and calicoctl for LKE clusters
36+
run: |
37+
curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl"
38+
curl -LO "https://github.com/projectcalico/calico/releases/download/v3.25.0/calicoctl-linux-amd64"
39+
chmod +x calicoctl-linux-amd64 kubectl
40+
mv calicoctl-linux-amd64 /usr/local/bin/calicoctl
41+
mv kubectl /usr/local/bin/kubectl
3742
3843
- name: Install Package
3944
run: make install
@@ -48,6 +53,13 @@ jobs:
4853
env:
4954
LINODE_CLI_TOKEN: ${{ secrets.LINODE_TOKEN }}
5055

56+
- name: Apply Calico Rules to LKE
57+
if: always()
58+
run: |
59+
cd scripts && ./lke_calico_rules_e2e.sh
60+
env:
61+
LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }}
62+
5163
- name: Upload test results
5264
if: always()
5365
run: |

‎scripts/lke-policy.yaml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
apiVersion: projectcalico.org/v3
2+
kind: GlobalNetworkPolicy
3+
metadata:
4+
name: lke-rules
5+
spec:
6+
preDNAT: true
7+
applyOnForward: true
8+
order: 100
9+
# Remember to run calicoctl patch command for this to work
10+
selector: ""
11+
ingress:
12+
# Allow ICMP
13+
- action: Allow
14+
protocol: ICMP
15+
- action: Allow
16+
protocol: ICMPv6
17+
18+
# Allow LKE-required ports
19+
- action: Allow
20+
protocol: TCP
21+
destination:
22+
nets:
23+
- 192.168.128.0/17
24+
- 10.0.0.0/8
25+
ports:
26+
- 10250
27+
- 10256
28+
- 179
29+
- action: Allow
30+
protocol: UDP
31+
destination:
32+
nets:
33+
- 192.168.128.0/17
34+
- 10.2.0.0/16
35+
ports:
36+
- 51820
37+
38+
# Allow NodeBalancer ingress to the Node Ports & Allow DNS
39+
- action: Allow
40+
protocol: TCP
41+
source:
42+
nets:
43+
- 192.168.255.0/24
44+
- 10.0.0.0/8
45+
destination:
46+
ports:
47+
- 53
48+
- 30000:32767
49+
- action: Allow
50+
protocol: UDP
51+
source:
52+
nets:
53+
- 192.168.255.0/24
54+
- 10.0.0.0/8
55+
destination:
56+
ports:
57+
- 53
58+
- 30000:32767
59+
60+
# Allow cluster internal communication
61+
- action: Allow
62+
destination:
63+
nets:
64+
- 10.0.0.0/8
65+
- action: Allow
66+
source:
67+
nets:
68+
- 10.0.0.0/8
69+
70+
# 127.0.0.1/32 is needed for kubectl exec and node-shell
71+
- action: Allow
72+
destination:
73+
nets:
74+
- 127.0.0.1/32
75+
76+
# Block everything else
77+
- action: Deny
78+
- action: Log

‎scripts/lke_calico_rules_e2e.sh‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/bin/bash
2+
3+
RETRIES=3
4+
DELAY=30
5+
6+
# Function to retry a command with exponential backoff
7+
retry_command() {
8+
local retries=$1
9+
local wait_time=60
10+
shift
11+
until "$@"; do
12+
if ((retries == 0)); then
13+
echo "Command failed after multiple retries. Exiting."
14+
exit 1
15+
fi
16+
echo "Command failed. Retrying in $wait_time seconds..."
17+
sleep $wait_time
18+
((retries--))
19+
wait_time=$((wait_time * 2))
20+
done
21+
}
22+
23+
# Fetch the list of LKE cluster IDs
24+
CLUSTER_IDS=$(curl -s -H "Authorization: Bearer $LINODE_TOKEN" \
25+
-H "Content-Type: application/json" \
26+
"https://api.linode.com/v4/lke/clusters" | jq -r '.data[].id')
27+
28+
# Check if CLUSTER_IDS is empty
29+
if [ -z "$CLUSTER_IDS" ]; then
30+
echo "All clusters have been cleaned and properly destroyed. No need to apply inbound or outbound rules"
31+
exit 0
32+
fi
33+
34+
for ID in $CLUSTER_IDS; do
35+
echo "Applying Calico rules to nodes in Cluster ID: $ID"
36+
37+
# Download cluster configuration file with retry
38+
for ((i=1; i<=RETRIES; i++)); do
39+
config_response=$(curl -sH "Authorization: Bearer $LINODE_TOKEN" "https://api.linode.com/v4/lke/clusters/$ID/kubeconfig")
40+
if [[ $config_response != *"kubeconfig is not yet available"* ]]; then
41+
echo $config_response | jq -r '.[] | @base64d' > "/tmp/${ID}_config.yaml"
42+
break
43+
fi
44+
echo "Attempt $i to download kubeconfig for cluster $ID failed. Retrying in $DELAY seconds..."
45+
sleep $DELAY
46+
done
47+
48+
if [[ $config_response == *"kubeconfig is not yet available"* ]]; then
49+
echo "kubeconfig for cluster id:$ID not available after $RETRIES attempts, mostly likely it is an empty cluster. Skipping..."
50+
else
51+
# Export downloaded config file
52+
export KUBECONFIG="/tmp/${ID}_config.yaml"
53+
54+
retry_command $RETRIES kubectl get nodes
55+
56+
retry_command $RETRIES calicoctl patch kubecontrollersconfiguration default --allow-version-mismatch --patch='{"spec": {"controllers": {"node": {"hostEndpoint": {"autoCreate": "Enabled"}}}}}'
57+
58+
retry_command $RETRIES calicoctl apply --allow-version-mismatch -f "$(pwd)/lke-policy.yaml"
59+
fi
60+
done

0 commit comments

Comments
 (0)