Skip to content

Commit cc45cf5

Browse files
committed
Resolve other necessary changes
1 parent 8cd987e commit cc45cf5

File tree

3 files changed

+183
-57
lines changed

3 files changed

+183
-57
lines changed

src/integration-tests/bash/cleanup.sh

Lines changed: 129 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,13 @@
2727
#
2828
# The test runs in 4 phases:
2929
#
30-
# Phase 1: Delete test kubernetes artifacts in an orderly
31-
# fashion via kubectl delete -f of previous tests's yaml
32-
# plus various direct kubectl deletes.
30+
# Phase 1: Delete test kubernetes artifacts with labels.
3331
#
34-
# Phase 2: Wait 15 seconds to see if stage 1 succeeded, and
35-
# if not, repeatedly search for all test related kubectl
36-
# artifacts and try delete them directly for up to 60 more
37-
# seconds. This phase has no dependency on the
38-
# previous test run's yaml files. It makes no
32+
# Phase 2: Wait 15 seconds to see if stage 1 succeeded, and
33+
# if not, repeatedly search for all test related kubectl
34+
# artifacts and try delete them directly for up to 60 more
35+
# seconds. This phase has no dependency on the
36+
# previous test run's yaml files. It makes no
3937
# attempt to delete artifacts in a particular order.
4038
#
4139
# Phase 3: Use a kubernetes job to delete the PV directories
@@ -47,13 +45,6 @@
4745
# see LEASE_ID above.
4846
#
4947

50-
DOMAINS=(domain1 domain2 domain3 domain4 domain5)
51-
DOMAIN_NAMESPACES=(default default test1 test2 default)
52-
DCOUNT=${#DOMAINS[@]}
53-
54-
OPER_NAMESPACES=(weblogic-operator-1 weblogic-operator-2)
55-
OCOUNT=${#OPER_NAMESPACES[@]}
56-
5748
SCRIPTPATH="$( cd "$(dirname "$0")" > /dev/null 2>&1 ; pwd -P )"
5849
PROJECT_ROOT="$SCRIPTPATH/../../.."
5950
RESULT_ROOT=${RESULT_ROOT:-/scratch/$USER/wl_k8s_test_results}
@@ -63,25 +54,133 @@ USER_PROJECTS_DIR="$RESULT_DIR/user-projects"
6354
TMP_DIR="$RESULT_DIR/cleanup_tmp"
6455
JOB_NAME="weblogic-command-job"
6556

57+
function fail {
58+
echo @@ cleanup.sh: Error "$@"
59+
exit 1
60+
}
61+
62+
#!/bin/bash
63+
#
64+
# Usage:
65+
# getResWithLabel outfilename
66+
#
67+
function getResWithLabel {
68+
69+
# first, let's get all namespaced types with -l $LABEL_SELECTOR
70+
kubectl get $NAMESPACED_TYPES \
71+
-l "$LABEL_SELECTOR" \
72+
-o=jsonpath='{range .items[*]}{.kind}{" "}{.metadata.name}{" -n "}{.metadata.namespace}{"\n"}{end}' \
73+
--all-namespaces=true >> $1
74+
75+
# now, get all non-namespaced types with -l $LABEL_SELECTOR
76+
kubectl get $NOT_NAMESPACED_TYPES \
77+
-l "$LABEL_SELECTOR" \
78+
-o=jsonpath='{range .items[*]}{.kind}{" "}{.metadata.name}{"\n"}{end}' \
79+
--all-namespaces=true >> $1
80+
}
81+
82+
#
83+
# Usage:
84+
# deleteResWithLabel outputfile
85+
#
86+
function deleteWithOneLabel {
87+
echo @@ Delete resources with label $LABEL_SELECTOR.
88+
# clean the output file first
89+
if [ -e $1 ]; then
90+
rm $1
91+
fi
92+
93+
echo @@ Deleting resources with label $LABEL_SELECTOR.
94+
getResWithLabel $1
95+
# delete namespaced types
96+
cat $1 | awk '{ print $4 }' | grep -v "^$" | sort -u | while read line; do
97+
kubectl -n $line delete $NAMESPACED_TYPES -l "$LABEL_SELECTOR"
98+
done
99+
100+
# delete non-namespaced types
101+
local no_namespace_count=`grep -c -v " -n " $1`
102+
if [ ! "$no_namespace_count" = "0" ]; then
103+
kubectl delete $NOT_NAMESPACED_TYPES -l "$LABEL_SELECTOR"
104+
fi
105+
106+
echo "@@ Waiting for pods to stop running."
107+
local total=0
108+
local mstart=`date +%s`
109+
local mnow=mstart
110+
local maxwaitsecs=60
111+
while [ $((mnow - mstart)) -lt $maxwaitsecs ]; do
112+
pods=($(kubectl get pods --all-namespaces -l $LABEL_SELECTOR -o jsonpath='{range .items[*]}{.metadata.name} {end}'))
113+
total=${#pods[*]}
114+
if [ $total -eq 0 ] ; then
115+
break
116+
else
117+
echo "@@ There are $total running pods with label $LABEL_SELECTOR."
118+
fi
119+
sleep 3
120+
mnow=`date +%s`
121+
done
122+
123+
if [ $total -gt 0 ]; then
124+
echo "Warning: after waiting $maxwaitsecs seconds, there are still $total running pods with label $LABEL_SELECTOR."
125+
fi
126+
}
127+
128+
#
129+
# Usage:
130+
# deleteNamespaces outputfile
131+
#
132+
function deleteNamespaces {
133+
cat $1 | awk '{ print $4 }' | grep -v "^$" | sort -u | while read line; do
134+
if [ "$line" != "default" ]; then
135+
kubectl delete namespace $line --ignore-not-found
136+
fi
137+
done
138+
139+
}
140+
141+
function deleteWithLabels {
142+
NAMESPACED_TYPES="pod,job,deploy,rs,service,pvc,ingress,cm,serviceaccount,role,rolebinding,secret"
143+
144+
DOMAIN_CRD="domains.weblogic.oracle"
145+
if [ `kubectl get crd $DOMAIN_CRD --ignore-not-found | grep $DOMAIN_CRD | wc -l` = 1 ]; then
146+
NAMESPACED_TYPES="$DOMAIN_CRD,$NAMESPACED_TYPES"
147+
fi
148+
149+
NOT_NAMESPACED_TYPES="pv,crd,clusterroles,clusterrolebindings"
150+
151+
tempfile="/tmp/$(basename $0).tmp.$$" # == /tmp/[script-file-name].tmp.[pid]
152+
153+
echo @@ Deleting domain resources.
154+
LABEL_SELECTOR="weblogic.domainUID"
155+
deleteWithOneLabel "$tempfile-0"
156+
157+
echo @@ Deleting wls operator resources.
158+
LABEL_SELECTOR="weblogic.operatorName"
159+
deleteWithOneLabel "$tempfile-1"
160+
161+
deleteNamespaces "$tempfile-0"
162+
deleteNamespaces "$tempfile-1"
163+
}
164+
66165
# function genericDelete
67166
#
68167
# This function is a 'generic kubernetes delete' that takes three arguments:
69168
#
70169
# arg1: Comma separated list of types of kubernetes namespaced types to search/delete.
71-
# example: "all,cm,pvc,ns,roles,rolebindings,secrets"
170+
# example: "all,cm,pvc,ns,roles,rolebindings,secrets"
72171
#
73172
# arg2: Comma separated list of types of kubernetes non-namespaced types to search/delete.
74-
# example: "crd,pv,clusterroles,clusterrolebindings"
173+
# example: "crd,pv,clusterroles,clusterrolebindings"
75174
#
76-
# arg3: '|' (pipe) separated list of keywords.
175+
# arg3: '|' (pipe) separated list of keywords.
77176
# Artifacts with a label or name that contains one
78177
# or more of the keywords are delete candidates.
79178
# example: "logstash|kibana|elastisearch|weblogic|elk|domain"
80179
#
81180
# It runs in two stages:
82181
# In the first, wait to see if artifacts delete on their own.
83182
# In the second, try to delete any leftovers.
84-
#
183+
#
85184
function genericDelete {
86185

87186
for iteration in first second; do
@@ -136,23 +235,23 @@ function genericDelete {
136235

137236
echo "@@ Waiting for $artcount_total artifacts to delete. Wait time $((mnow - mstart)) seconds (max=$maxwaitsecs). Waiting for:"
138237

139-
cat $resfile_yes | awk '{ print "n=" $1 " " $2 }'
238+
cat $resfile_yes | awk '{ print "n=" $1 " " $2 }'
140239
cat $resfile_no | awk '{ print $1 }'
141240

142241
else
143242
# in the second thirty seconds we try to delete remaining artifacts
144243

145244
echo "@@ Trying to delete ${artcount_total} leftover artifacts, including ${artcount_yes} namespaced artifacts and ${artcount_no} non-namespaced artifacts, wait time $((mnow - mstart)) seconds (max=$maxwaitsecs)."
146245

147-
if [ ${artcount_yes} -gt 0 ]; then
246+
if [ ${artcount_yes} -gt 0 ]; then
148247
cat "$resfile_yes" | while read line; do
149248
local args="`echo \"$line\" | awk '{ print "-n " $1 " delete " $2 " --ignore-not-found" }'`"
150249
echo "kubectl $args"
151250
kubectl $args
152251
done
153252
fi
154253

155-
if [ ${artcount_no} -gt 0 ]; then
254+
if [ ${artcount_no} -gt 0 ]; then
156255
cat "$resfile_no" | while read line; do
157256
echo "kubectl delete $line --ignore-not-found"
158257
kubectl delete $line --ignore-not-found
@@ -308,24 +407,27 @@ function fail {
308407
}
309408

310409
echo @@ Starting cleanup.
410+
script="${BASH_SOURCE[0]}"
411+
scriptDir="$( cd "$(dirname "${script}")" > /dev/null 2>&1 ; pwd -P)"
311412

312413
echo "@@ RESULT_ROOT=$RESULT_ROOT TMP_DIR=$TMP_DIR RESULT_DIR=$RESULT_DIR PROJECT_ROOT=$PROJECT_ROOT"
313414

314415
mkdir -p $TMP_DIR || fail No permision to create directory $TMP_DIR
315416

316-
# try an ordered/controlled delete first
317-
318-
#Comment out orderlyDelete so we can fully test generic delete (do not merge this change!)
319-
orderlyDelete
417+
# first, try to delete with labels since the conversion is that all created resources need to
418+
# have the proper label(s)
419+
echo @@ Starting deleteWithLabels
420+
deleteWithLabels
320421

321-
# try a generic delete in case the orderly delete missed something, this runs in two phases:
422+
# second, try a generic delete in case there are some leftover resources, this runs in two phases:
322423
# phase 1: wait to see if artifacts dissappear naturally due to the above orderlyDelete
323424
# phase 2: kubectl delete left over artifacts
324425
# arguments
325426
# arg1 - namespaced kubernetes artifacts
326427
# arg2 - non-namespaced artifacts
327428
# arg3 - keywords in deletable artificats
328429

430+
echo @@ Starting genericDelete
329431
genericDelete "all,cm,pvc,roles,rolebindings,serviceaccount,secrets" "crd,pv,ns,clusterroles,clusterrolebindings" "logstash|kibana|elastisearch|weblogic|elk|domain|traefik"
330432
SUCCESS="$?"
331433

0 commit comments

Comments
 (0)