Skip to content

Commit 521dc95

Browse files
joekrjoekr
authored
add argo e2e workflow (#477)
This simple workflow isn't perfect, but it replaces the need to manually run the e2e tests locally or on a VM. This makes it more repeatable across the team.
1 parent f6e9903 commit 521dc95

File tree

3 files changed

+569
-1
lines changed

3 files changed

+569
-1
lines changed

‎argo/capoci-e2e-workflow.yaml‎

Lines changed: 368 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,368 @@
1+
# Argo Workflow to run CAPOCI e2e tests via scripts/ci-e2e.sh
2+
# - Uses docker-in-docker (dind) sidecar so the Makefile can docker build/push
3+
# - Installs minimal tooling in the main container (git, make, curl, openssh-client, docker CLI)
4+
# - Clones this repository and invokes ./scripts/ci-e2e.sh (which does the build and testing)
5+
# - Maps parameters to environment variables consumed by the script and e2e config
6+
#
7+
# Submit directly (no template install needed):
8+
# argo submit argo/capoci-e2e-workflow.yaml \
9+
# -p oci_compartment_id=ocid1.compartment.oc1..example \
10+
# -p oci_image_id=ocid1.image.oc1..example \
11+
# -p oci_oracle_linux_image_id=ocid1.image.oc1..example \
12+
# -p oci_upgrade_image_id=ocid1.image.oc1..example \
13+
# -p oci_managed_node_image_id=ocid1.image.oc1..optional \
14+
# -p oci_alternative_region_image_id=ocid1.image.oc1..example \
15+
# -p registry=ghcr.io/your-org \
16+
# -p use_instance_principal=true \
17+
# -p use_instance_principal_b64=dHJ1ZQ== \
18+
# -p oci_ssh_key="ssh-rsa AAAA... your-key"
19+
#
20+
# Notes:
21+
# - USE_INSTANCE_PRINCIPAL_B64 commonly needs "dHJ1ZQ==" for true, "ZmFsc2U=" for false (base64).
22+
# - If OCI_SSH_KEY is omitted, the script will generate a temporary key.
23+
# - REGISTRY defaults to ghcr.io/oracle; override if pushing to your org's registry.
24+
# - git_ref defaults to main
25+
# - git_repo defaults to https://github.com/oracle/cluster-api-provider-oci.git
26+
# - ginkgo_focus can be used to run a single test
27+
28+
apiVersion: argoproj.io/v1alpha1
29+
kind: Workflow
30+
metadata:
31+
namespace: argo
32+
generateName: capoci-e2e-
33+
spec:
34+
entrypoint: run
35+
serviceAccountName: argo
36+
parallelism: 1
37+
podGC:
38+
strategy: OnWorkflowSuccess
39+
ttlStrategy:
40+
secondsAfterSuccess: 86400
41+
secondsAfterFailure: 604800
42+
43+
# Default parameter values (override at submit time)
44+
arguments:
45+
parameters:
46+
- name: git_repo
47+
value: https://github.com/oracle/cluster-api-provider-oci.git
48+
- name: git_ref
49+
value: main
50+
51+
- name: registry
52+
value: ghcr.io/oracle
53+
54+
# REQUIRED by scripts/ci-e2e.sh
55+
- name: oci_compartment_id
56+
value: ""
57+
- name: oci_image_id
58+
value: ""
59+
- name: oci_oracle_linux_image_id
60+
value: ""
61+
- name: oci_upgrade_image_id
62+
value: ""
63+
- name: oci_alternative_region_image_id
64+
value: ""
65+
- name: oci_managed_node_image_id
66+
value: ""
67+
68+
# Optional image IDs
69+
- name: oci_windows_image_id
70+
value: ""
71+
72+
# Optional SSH public key. If empty, the script generates one.
73+
- name: oci_ssh_key
74+
value: ""
75+
76+
# Feature/behavior toggles
77+
- name: cluster_topology
78+
value: "true"
79+
- name: exp_machine_pool
80+
value: "true"
81+
- name: exp_oke
82+
value: "false"
83+
- name: use_instance_principal
84+
value: "false"
85+
# Base64 of "true" or "false". Common values: true=dHJ1ZQ==, false=ZmFsc2U=
86+
- name: use_instance_principal_b64
87+
value: "dHJ1ZQ=="
88+
- name: oci_alternative_region
89+
value: "us-sanjose-1"
90+
91+
# Test execution tuning
92+
- name: ginkgo_nodes
93+
value: "3"
94+
- name: ginkgo_focus
95+
value: ""
96+
- name: kind_node_image
97+
value: "kindest/node:v1.29.6"
98+
- name: tag
99+
value: ""
100+
- name: node_machine_count
101+
value: "1"
102+
- name: ocir_username
103+
value: ""
104+
- name: ocir_region
105+
value: "us-phoenix-1"
106+
- name: ocir_region_short_code
107+
value: "phx"
108+
109+
templates:
110+
- name: run
111+
outputs:
112+
artifacts:
113+
- name: e2e-report
114+
path: /workspace/report.json
115+
optional: true
116+
- name: e2e-logs
117+
path: /workspace/_artifacts_logs_only
118+
optional: true
119+
- name: e2e-artifacts
120+
path: /workspace/_artifacts
121+
optional: true
122+
volumes:
123+
- name: workspace
124+
emptyDir: {}
125+
- name: dind-storage
126+
emptyDir: {}
127+
128+
sidecars:
129+
- name: dind
130+
image: docker.io/library/docker:24-dind
131+
args:
132+
- "--host=tcp://0.0.0.0:2375"
133+
- "--host=unix:///var/run/docker.sock"
134+
env:
135+
- name: DOCKER_TLS_CERTDIR
136+
value: ""
137+
readinessProbe:
138+
tcpSocket:
139+
port: 2375
140+
initialDelaySeconds: 2
141+
periodSeconds: 2
142+
securityContext:
143+
privileged: true
144+
resources:
145+
requests:
146+
cpu: "4"
147+
memory: "8Gi"
148+
limits:
149+
cpu: "8"
150+
memory: "16Gi"
151+
volumeMounts:
152+
- name: dind-storage
153+
mountPath: /var/lib/docker
154+
155+
container:
156+
image: docker.io/library/golang:1.23-bookworm
157+
workingDir: /workspace
158+
securityContext:
159+
runAsUser: 0
160+
resources:
161+
requests:
162+
cpu: "4"
163+
memory: "8Gi"
164+
limits:
165+
cpu: "8"
166+
memory: "16Gi"
167+
env:
168+
# Docker socket via dind sidecar
169+
- name: DOCKER_HOST
170+
value: tcp://localhost:2375
171+
172+
# Registry for docker build/push from the Makefile
173+
- name: REGISTRY
174+
value: "{{workflow.parameters.registry}}"
175+
176+
# Required envs for scripts/ci-e2e.sh
177+
- name: OCI_COMPARTMENT_ID
178+
value: "{{workflow.parameters.oci_compartment_id}}"
179+
- name: OCI_IMAGE_ID
180+
value: "{{workflow.parameters.oci_image_id}}"
181+
- name: OCI_ORACLE_LINUX_IMAGE_ID
182+
value: "{{workflow.parameters.oci_oracle_linux_image_id}}"
183+
- name: OCI_UPGRADE_IMAGE_ID
184+
value: "{{workflow.parameters.oci_upgrade_image_id}}"
185+
- name: OCI_MANAGED_NODE_IMAGE_ID
186+
value: "{{workflow.parameters.oci_managed_node_image_id}}"
187+
# Also set the variant used by e2e_conf.yaml envsubst.
188+
- name: KUBERNETES_UPGRADE_OCI_IMAGE_ID
189+
value: "{{workflow.parameters.oci_upgrade_image_id}}"
190+
- name: OCI_ALTERNATIVE_REGION_IMAGE_ID
191+
value: "{{workflow.parameters.oci_alternative_region_image_id}}"
192+
193+
# Optional envs used by templates/config
194+
- name: OCI_WINDOWS_IMAGE_ID
195+
value: "{{workflow.parameters.oci_windows_image_id}}"
196+
- name: OCI_SSH_KEY
197+
value: "{{workflow.parameters.oci_ssh_key}}"
198+
- name: OCI_ALTERNATIVE_REGION
199+
value: "{{workflow.parameters.oci_alternative_region}}"
200+
201+
# Feature gates and behavior flags
202+
- name: CLUSTER_TOPOLOGY
203+
value: "{{workflow.parameters.cluster_topology}}"
204+
- name: EXP_MACHINE_POOL
205+
value: "{{workflow.parameters.exp_machine_pool}}"
206+
- name: EXP_OKE
207+
value: "{{workflow.parameters.exp_oke}}"
208+
- name: USE_INSTANCE_PRINCIPAL
209+
value: "{{workflow.parameters.use_instance_principal}}"
210+
- name: USE_INSTANCE_PRINCIPAL_B64
211+
value: "{{workflow.parameters.use_instance_principal_b64}}"
212+
213+
# Test runner config
214+
- name: GINKGO_NODES
215+
value: "{{workflow.parameters.ginkgo_nodes}}"
216+
- name: GINKGO_FOCUS_PARAM
217+
value: "{{workflow.parameters.ginkgo_focus}}"
218+
- name: KIND_NODE_IMAGE
219+
value: "{{workflow.parameters.kind_node_image}}"
220+
- name: KIND_IMAGE
221+
value: "{{workflow.parameters.kind_node_image}}"
222+
- name: KIND_CLUSTER_IMAGE
223+
value: "{{workflow.parameters.kind_node_image}}"
224+
- name: KIND_EXPERIMENTAL_IMAGE
225+
value: "{{workflow.parameters.kind_node_image}}"
226+
- name: TAG
227+
value: "{{workflow.parameters.tag}}"
228+
- name: NODE_MACHINE_COUNT
229+
value: "{{workflow.parameters.node_machine_count}}"
230+
- name: OCIR_USERNAME
231+
value: "{{workflow.parameters.ocir_username}}"
232+
- name: OCIR_REGION
233+
value: "{{workflow.parameters.ocir_region}}"
234+
- name: OCIR_REGION_SHORT_CODE
235+
value: "{{workflow.parameters.ocir_region}}"
236+
- name: KIND_CGROUP_DRIVER
237+
value: "cgroupfs"
238+
239+
command: ["bash", "-ec"]
240+
args:
241+
- |
242+
set -o errexit
243+
set -o nounset
244+
set -o pipefail
245+
246+
# Base tooling for the build and e2e scripts
247+
apt-get update
248+
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
249+
git make curl ca-certificates openssh-client jq
250+
update-ca-certificates
251+
252+
# Install docker CLI to talk to dind sidecar
253+
# (use static binary to avoid extra package repos)
254+
curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-24.0.9.tgz \
255+
| tar -xz -C /usr/local/bin --strip-components=1 docker/docker
256+
for i in {1..60}; do
257+
if docker version >/dev/null 2>&1; then
258+
break
259+
fi
260+
echo "waiting for docker daemon on ${DOCKER_HOST} ..."
261+
sleep 2
262+
done
263+
docker version
264+
265+
# Ensure binaries installed by the Makefile are on PATH (kustomize, ginkgo, etc.)
266+
export PATH="/workspace/bin:${PATH}"
267+
# Also provide kustomize in a standard location for tools that exec 'kustomize' by name.
268+
ln -sf /workspace/bin/kustomize /usr/local/bin/kustomize || true
269+
ln -sf /workspace/bin/ginkgo /usr/local/bin/ginkgo || true
270+
ln -sf /workspace/bin/kubectl /usr/local/bin/kubectl || true
271+
command -v kustomize || true
272+
kustomize version || true
273+
command -v kubectl || true
274+
kubectl version --client=true --short || true
275+
276+
# Install OCI CLI for docker authentication to OCIR
277+
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3 python3-venv python3-pip unzip curl jq
278+
export PATH="/root/.local/bin:${PATH}"
279+
python3 -m venv /opt/oci-cli
280+
/opt/oci-cli/bin/pip install --upgrade pip
281+
/opt/oci-cli/bin/pip install "oci_cli==3.71.1"
282+
ln -sf /opt/oci-cli/bin/oci /usr/local/bin/oci
283+
284+
oci --version || true
285+
286+
# Setup OCIR token and login
287+
echo "Generating the ocir token"
288+
echo "https://{{workflow.parameters.ocir_region}}.ocir.io/20180419/docker/token"
289+
OCIR_TOKEN=$(oci raw-request \
290+
--http-method GET \
291+
--target-uri "https://{{workflow.parameters.ocir_region}}.ocir.io/20180419/docker/token" \
292+
--auth instance_principal \
293+
--region "{{workflow.parameters.ocir_region}}" \
294+
--profile DEFAULT | jq -r .data.token)
295+
296+
echo "docker login"
297+
echo "${OCIR_TOKEN}" | docker login -u BEARER_TOKEN --password-stdin "{{workflow.parameters.ocir_region_short_code}}.ocir.io"
298+
299+
# Clone repository and checkout ref
300+
git clone --depth 1 --branch "{{workflow.parameters.git_ref}}" "{{workflow.parameters.git_repo}}" /workspace
301+
302+
# Optional Ginkgo focus override
303+
if [ -n "${GINKGO_FOCUS_PARAM:-}" ]; then
304+
export GINKGO_FOCUS="${GINKGO_FOCUS_PARAM}"
305+
fi
306+
307+
# Preflight: show which Kind node image will be used
308+
echo "KIND_IMAGE=${KIND_IMAGE:-unset} KIND_NODE_IMAGE=${KIND_NODE_IMAGE:-unset} KIND_CLUSTER_IMAGE=${KIND_CLUSTER_IMAGE:-unset} KIND_EXPERIMENTAL_IMAGE=${KIND_EXPERIMENTAL_IMAGE:-unset}"
309+
310+
# Start capoci-controller-manager log watcher in background (waits until 'kind' exists)
311+
export CLUSTER_NAME="${CLUSTER_NAME:-capoci-e2e}"
312+
export OUTPUT_DIR="/workspace/_artifacts"
313+
( until command -v kind >/dev/null 2>&1; do echo "waiting for kind binary..."; sleep 2; done; \
314+
./scripts/watch-capoci-controller.sh -n "${CLUSTER_NAME}" -o "${OUTPUT_DIR}" -r 180 -d 5 ) \
315+
& echo $! > /tmp/capoci_watch_pid
316+
317+
# Run the e2e test harness
318+
# This builds and runs the actual test. Everything else is package install and setup or reporting
319+
./scripts/ci-e2e.sh
320+
321+
# Stop watcher if running
322+
if [ -f /tmp/capoci_watch_pid ]; then
323+
kill "$(cat /tmp/capoci_watch_pid)" >/dev/null 2>&1 || true
324+
rm -f /tmp/capoci_watch_pid
325+
fi
326+
327+
# Build logs-only artifacts (logs + report.json), exclude YAML and other files
328+
mkdir -p /workspace/_artifacts_logs_only
329+
if [ -d /workspace/_artifacts ]; then
330+
find /workspace/_artifacts -type f -name '*.log' -exec cp --parents {} /workspace/_artifacts_logs_only \; || true
331+
fi
332+
if [ -f /workspace/report.json ]; then
333+
cp /workspace/report.json /workspace/_artifacts_logs_only/ || true
334+
fi
335+
336+
# Emit the ginkgo JSON report into logs for easy retrieval
337+
if [ -f /workspace/report.json ]; then
338+
echo "=== BEGIN report.json ==="
339+
cat /workspace/report.json || true
340+
echo "=== END report.json ==="
341+
fi
342+
343+
# Best-effort: surface CAPOCI controller logs from the kind mgmt cluster (if still present)
344+
if command -v kubectl >/dev/null 2>&1; then
345+
echo "=== capoci-controller-manager logs (cluster-api-provider-oci-system) ==="
346+
LOG_DIR="/workspace/_artifacts/capoci-controller-manager"
347+
mkdir -p "${LOG_DIR}"
348+
# Save pod listing and logs to files while also emitting to stdout
349+
kubectl --context kind-capoci-e2e -n cluster-api-provider-oci-system get pods -o wide | tee "${LOG_DIR}/pods.txt" || true
350+
pods=$(kubectl --context kind-capoci-e2e -n cluster-api-provider-oci-system get pods -l app=capoci-controller-manager -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true)
351+
for p in $pods; do
352+
echo "--- logs for pod: $p"
353+
kubectl --context kind-capoci-e2e -n cluster-api-provider-oci-system logs "$p" --all-containers=true --tail=-1 | tee "${LOG_DIR}/${p}.log" || true
354+
done
355+
if [ -z "${pods}" ]; then
356+
# Fallback to Deployment name if label not present
357+
kubectl --context kind-capoci-e2e -n cluster-api-provider-oci-system logs deploy/capoci-controller-manager --all-containers=true --tail=-1 | tee "${LOG_DIR}/deployment.log" || true
358+
fi
359+
fi
360+
361+
# Rebuild logs-only artifacts to include controller logs saved above
362+
if [ -d /workspace/_artifacts ]; then
363+
find /workspace/_artifacts -type f -name '*.log' -exec cp --parents {} /workspace/_artifacts_logs_only \; || true
364+
fi
365+
366+
volumeMounts:
367+
- name: workspace
368+
mountPath: /workspace

‎hack/ensure-kind.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ set -o nounset
66
set -o pipefail
77

88
GOPATH_BIN="$(go env GOPATH)/bin/"
9-
MINIMUM_KIND_VERSION=v0.10.0
9+
MINIMUM_KIND_VERSION=v0.25.0
1010

1111
# Ensure the kind tool exists and is a viable version, or installs it
1212
verify_kind_version() {
@@ -39,3 +39,5 @@ EOF
3939
}
4040

4141
verify_kind_version
42+
# Print kind version for visibility in CI logs
43+
kind version || true

0 commit comments

Comments
 (0)