27
27
#
28
28
# The test runs in 4 phases:
29
29
#
30
- # Phase 1: Delete test kubernetes artifacts in an orderly
31
- # fashion via kubectl delete -f of previous tests's yaml
32
- # plus various direct kubectl deletes.
30
+ # Phase 1: Delete test kubernetes artifacts with labels.
33
31
#
34
- # Phase 2: Wait 15 seconds to see if stage 1 succeeded, and
35
- # if not, repeatedly search for all test related kubectl
36
- # artifacts and try delete them directly for up to 60 more
37
- # seconds. This phase has no dependency on the
38
- # previous test run's yaml files. It makes no
32
+ # Phase 2: Wait 15 seconds to see if stage 1 succeeded, and
33
+ # if not, repeatedly search for all test related kubectl
34
+ # artifacts and try delete them directly for up to 60 more
35
+ # seconds. This phase has no dependency on the
36
+ # previous test run's yaml files. It makes no
39
37
# attempt to delete artifacts in a particular order.
40
38
#
41
39
# Phase 3: Use a kubernetes job to delete the PV directories
47
45
# see LEASE_ID above.
48
46
#
49
47
50
- DOMAINS=(domain1 domain2 domain3 domain4 domain5)
51
- DOMAIN_NAMESPACES=(default default test1 test2 default)
52
- DCOUNT=${# DOMAINS[@]}
53
-
54
- OPER_NAMESPACES=(weblogic-operator-1 weblogic-operator-2)
55
- OCOUNT=${# OPER_NAMESPACES[@]}
56
-
57
48
SCRIPTPATH=" $( cd " $( dirname " $0 " ) " > /dev/null 2>&1 ; pwd -P ) "
58
49
PROJECT_ROOT=" $SCRIPTPATH /../../.."
59
50
RESULT_ROOT=${RESULT_ROOT:-/ scratch/ $USER / wl_k8s_test_results}
@@ -63,25 +54,133 @@ USER_PROJECTS_DIR="$RESULT_DIR/user-projects"
63
54
TMP_DIR=" $RESULT_DIR /cleanup_tmp"
64
55
JOB_NAME=" weblogic-command-job"
65
56
57
+ function fail {
58
+ echo @@ cleanup.sh: Error " $@ "
59
+ exit 1
60
+ }
61
+
62
+ #! /bin/bash
63
+ #
64
+ # Usage:
65
+ # getResWithLabel outfilename
66
+ #
67
+ function getResWithLabel {
68
+
69
+ # first, let's get all namespaced types with -l $LABEL_SELECTOR
70
+ kubectl get $NAMESPACED_TYPES \
71
+ -l " $LABEL_SELECTOR " \
72
+ -o=jsonpath=' {range .items[*]}{.kind}{" "}{.metadata.name}{" -n "}{.metadata.namespace}{"\n"}{end}' \
73
+ --all-namespaces=true >> $1
74
+
75
+ # now, get all non-namespaced types with -l $LABEL_SELECTOR
76
+ kubectl get $NOT_NAMESPACED_TYPES \
77
+ -l " $LABEL_SELECTOR " \
78
+ -o=jsonpath=' {range .items[*]}{.kind}{" "}{.metadata.name}{"\n"}{end}' \
79
+ --all-namespaces=true >> $1
80
+ }
81
+
82
+ #
83
+ # Usage:
84
+ # deleteResWithLabel outputfile
85
+ #
86
+ function deleteWithOneLabel {
87
+ echo @@ Delete resources with label $LABEL_SELECTOR .
88
+ # clean the output file first
89
+ if [ -e $1 ]; then
90
+ rm $1
91
+ fi
92
+
93
+ echo @@ Deleting resources with label $LABEL_SELECTOR .
94
+ getResWithLabel $1
95
+ # delete namespaced types
96
+ cat $1 | awk ' { print $4 }' | grep -v " ^$" | sort -u | while read line; do
97
+ kubectl -n $line delete $NAMESPACED_TYPES -l " $LABEL_SELECTOR "
98
+ done
99
+
100
+ # delete non-namespaced types
101
+ local no_namespace_count=` grep -c -v " -n " $1 `
102
+ if [ ! " $no_namespace_count " = " 0" ]; then
103
+ kubectl delete $NOT_NAMESPACED_TYPES -l " $LABEL_SELECTOR "
104
+ fi
105
+
106
+ echo " @@ Waiting for pods to stop running."
107
+ local total=0
108
+ local mstart=` date +%s`
109
+ local mnow=mstart
110
+ local maxwaitsecs=60
111
+ while [ $(( mnow - mstart)) -lt $maxwaitsecs ]; do
112
+ pods=($( kubectl get pods --all-namespaces -l $LABEL_SELECTOR -o jsonpath=' {range .items[*]}{.metadata.name} {end}' ) )
113
+ total=${# pods[*]}
114
+ if [ $total -eq 0 ] ; then
115
+ break
116
+ else
117
+ echo " @@ There are $total running pods with label $LABEL_SELECTOR ."
118
+ fi
119
+ sleep 3
120
+ mnow=` date +%s`
121
+ done
122
+
123
+ if [ $total -gt 0 ]; then
124
+ echo " Warning: after waiting $maxwaitsecs seconds, there are still $total running pods with label $LABEL_SELECTOR ."
125
+ fi
126
+ }
127
+
128
+ #
129
+ # Usage:
130
+ # deleteNamespaces outputfile
131
+ #
132
+ function deleteNamespaces {
133
+ cat $1 | awk ' { print $4 }' | grep -v " ^$" | sort -u | while read line; do
134
+ if [ " $line " != " default" ]; then
135
+ kubectl delete namespace $line --ignore-not-found
136
+ fi
137
+ done
138
+
139
+ }
140
+
141
+ function deleteWithLabels {
142
+ NAMESPACED_TYPES=" pod,job,deploy,rs,service,pvc,ingress,cm,serviceaccount,role,rolebinding,secret"
143
+
144
+ DOMAIN_CRD=" domains.weblogic.oracle"
145
+ if [ ` kubectl get crd $DOMAIN_CRD --ignore-not-found | grep $DOMAIN_CRD | wc -l` = 1 ]; then
146
+ NAMESPACED_TYPES=" $DOMAIN_CRD ,$NAMESPACED_TYPES "
147
+ fi
148
+
149
+ NOT_NAMESPACED_TYPES=" pv,crd,clusterroles,clusterrolebindings"
150
+
151
+ tempfile=" /tmp/$( basename $0 ) .tmp.$$ " # == /tmp/[script-file-name].tmp.[pid]
152
+
153
+ echo @@ Deleting domain resources.
154
+ LABEL_SELECTOR=" weblogic.domainUID"
155
+ deleteWithOneLabel " $tempfile -0"
156
+
157
+ echo @@ Deleting wls operator resources.
158
+ LABEL_SELECTOR=" weblogic.operatorName"
159
+ deleteWithOneLabel " $tempfile -1"
160
+
161
+ deleteNamespaces " $tempfile -0"
162
+ deleteNamespaces " $tempfile -1"
163
+ }
164
+
66
165
# function genericDelete
67
166
#
68
167
# This function is a 'generic kubernetes delete' that takes three arguments:
69
168
#
70
169
# arg1: Comma separated list of types of kubernetes namespaced types to search/delete.
71
- # example: "all,cm,pvc,ns,roles,rolebindings,secrets"
170
+ # example: "all,cm,pvc,ns,roles,rolebindings,secrets"
72
171
#
73
172
# arg2: Comma separated list of types of kubernetes non-namespaced types to search/delete.
74
- # example: "crd,pv,clusterroles,clusterrolebindings"
173
+ # example: "crd,pv,clusterroles,clusterrolebindings"
75
174
#
76
- # arg3: '|' (pipe) separated list of keywords.
175
+ # arg3: '|' (pipe) separated list of keywords.
77
176
# Artifacts with a label or name that contains one
78
177
# or more of the keywords are delete candidates.
79
178
# example: "logstash|kibana|elastisearch|weblogic|elk|domain"
80
179
#
81
180
# It runs in two stages:
82
181
# In the first, wait to see if artifacts delete on their own.
83
182
# In the second, try to delete any leftovers.
84
- #
183
+ #
85
184
function genericDelete {
86
185
87
186
for iteration in first second; do
@@ -136,23 +235,23 @@ function genericDelete {
136
235
137
236
echo " @@ Waiting for $artcount_total artifacts to delete. Wait time $(( mnow - mstart)) seconds (max=$maxwaitsecs ). Waiting for:"
138
237
139
- cat $resfile_yes | awk ' { print "n=" $1 " " $2 }'
238
+ cat $resfile_yes | awk ' { print "n=" $1 " " $2 }'
140
239
cat $resfile_no | awk ' { print $1 }'
141
240
142
241
else
143
242
# in the second thirty seconds we try to delete remaining artifacts
144
243
145
244
echo " @@ Trying to delete ${artcount_total} leftover artifacts, including ${artcount_yes} namespaced artifacts and ${artcount_no} non-namespaced artifacts, wait time $(( mnow - mstart)) seconds (max=$maxwaitsecs )."
146
245
147
- if [ ${artcount_yes} -gt 0 ]; then
246
+ if [ ${artcount_yes} -gt 0 ]; then
148
247
cat " $resfile_yes " | while read line; do
149
248
local args=" ` echo \" $line \" | awk ' { print "-n " $1 " delete " $2 " --ignore-not-found" }' ` "
150
249
echo " kubectl $args "
151
250
kubectl $args
152
251
done
153
252
fi
154
253
155
- if [ ${artcount_no} -gt 0 ]; then
254
+ if [ ${artcount_no} -gt 0 ]; then
156
255
cat " $resfile_no " | while read line; do
157
256
echo " kubectl delete $line --ignore-not-found"
158
257
kubectl delete $line --ignore-not-found
@@ -308,24 +407,27 @@ function fail {
308
407
}
309
408
310
409
echo @@ Starting cleanup.
410
+ script=" ${BASH_SOURCE[0]} "
411
+ scriptDir=" $( cd " $( dirname " ${script} " ) " > /dev/null 2>&1 ; pwd -P) "
311
412
312
413
echo " @@ RESULT_ROOT=$RESULT_ROOT TMP_DIR=$TMP_DIR RESULT_DIR=$RESULT_DIR PROJECT_ROOT=$PROJECT_ROOT "
313
414
314
415
mkdir -p $TMP_DIR || fail No permision to create directory $TMP_DIR
315
416
316
- # try an ordered/controlled delete first
317
-
318
- # Comment out orderlyDelete so we can fully test generic delete (do not merge this change!)
319
- orderlyDelete
417
+ # first, try to delete with labels since the conversion is that all created resources need to
418
+ # have the proper label(s)
419
+ echo @@ Starting deleteWithLabels
420
+ deleteWithLabels
320
421
321
- # try a generic delete in case the orderly delete missed something , this runs in two phases:
422
+ # second, try a generic delete in case there are some leftover resources , this runs in two phases:
322
423
# phase 1: wait to see if artifacts dissappear naturally due to the above orderlyDelete
323
424
# phase 2: kubectl delete left over artifacts
324
425
# arguments
325
426
# arg1 - namespaced kubernetes artifacts
326
427
# arg2 - non-namespaced artifacts
327
428
# arg3 - keywords in deletable artificats
328
429
430
+ echo @@ Starting genericDelete
329
431
genericDelete " all,cm,pvc,roles,rolebindings,serviceaccount,secrets" " crd,pv,ns,clusterroles,clusterrolebindings" " logstash|kibana|elastisearch|weblogic|elk|domain|traefik"
330
432
SUCCESS=" $? "
331
433
0 commit comments