Skip to content

Commit a195e8d

Browse files
PureWeenivanpovazan
PureWeen
and
ivanpovazan
authored
Remove cleanup for CI Builds (#20395)
* Remove cleanup for CI Builds * - added scripts * Update ios.cake * Update clean-bot.sh * add clean bot to yaml * - fix yml * - fix yaml * - try harder * - chmod * - update * - chmod * Update ios.cake * - add grep logging * - grep * - logs * Update device-tests-steps.yml * Update clean-bot.sh * Update device-tests-steps.yml * - zip up logs * - fix condition * - params * - test * - fix * - again * - test * - zippity * - zip * - fix condition * - log dir * - fix tabs * - diag reports * Try resetting simulators before test runs * Fix cmdline arguments for reset-simulator command --------- Co-authored-by: Ivan Povazan <ivan.povazan@gmail.com>
1 parent abcc7c3 commit a195e8d

File tree

6 files changed

+230
-3
lines changed

6 files changed

+230
-3
lines changed

‎eng/devices/ios.cake

+15-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ string DOTNET_PLATFORM = TEST_DEVICE.ToLower().Contains("simulator") ?
3838
$"iossimulator-{System.Runtime.InteropServices.RuntimeInformation.OSArchitecture.ToString().ToLower()}"
3939
: $"ios-arm64";
4040
string CONFIGURATION = Argument("configuration", "Debug");
41-
bool DEVICE_CLEANUP = Argument("cleanup", true);
41+
bool DEVICE_CLEANUP = Argument("cleanup", !IsCIBuild());
4242
string TEST_FRAMEWORK = "net472";
4343

4444
Information("Project File: {0}", PROJECT);
@@ -77,6 +77,7 @@ Setup(context =>
7777
if (TEST_DEVICE.IndexOf("_") != -1)
7878
{
7979
GetSimulators(TEST_DEVICE);
80+
ResetSimulators(TEST_DEVICE);
8081
}
8182
});
8283

@@ -480,6 +481,19 @@ void GetSimulators(string version)
480481
});
481482
}
482483

484+
void ResetSimulators(string version)
485+
{
486+
var logDirectory = GetLogDirectory();
487+
DotNetTool("tool", new DotNetToolSettings {
488+
ToolPath = DOTNET_TOOL_PATH,
489+
DiagnosticOutput = true,
490+
ArgumentCustomization = args => args.Append("run xharness apple simulators reset-simulator " +
491+
$"--output-directory=\"{logDirectory}\" " +
492+
$"--target=\"{version}\" " +
493+
$"--verbosity=\"Debug\" ")
494+
});
495+
}
496+
483497
void GetDevices(string version)
484498
{
485499
var deviceUdid = "";

‎eng/pipelines/common/device-tests-steps.yml

+17
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,14 @@ parameters:
1313
rebootAgent: true
1414

1515
steps:
16+
- ${{ if eq(parameters.platform, 'ios')}}:
17+
- bash: |
18+
chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
19+
chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-simulator-runtime.sh
20+
$(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
21+
displayName: 'Clean bot'
22+
continueOnError: true
23+
timeoutInMinutes: 60
1624
- template: provision.yml
1725
parameters:
1826
${{ if eq(parameters.platform, 'windows')}}:
@@ -86,6 +94,15 @@ steps:
8694
condition: and(succeeded(), eq(variables['Platform.Name'], 'Mac'))
8795
retryCountOnTaskFailure: 2
8896
97+
98+
- ${{ if eq(parameters.platform, 'ios')}}:
99+
- bash: |
100+
zip -9r "$(LogDirectory)/CoreSimulatorLog.zip" "$HOME/Library/Logs/CoreSimulator/"
101+
zip -9r "$(LogDirectory)/DiagnosticReports.zip" "$HOME/Library/Logs/DiagnosticReports/"
102+
displayName: Zip Simulator Logs
103+
condition: always()
104+
continueOnError: true
105+
89106
- task: PublishTestResults@2
90107
displayName: Publish the $(Agent.JobName) test results
91108
condition: always()

‎eng/pipelines/common/ui-tests-compatibility-steps.yml

+16-1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,14 @@ parameters:
1313
provisionPlatform: "windows"
1414

1515
steps:
16+
- bash: |
17+
chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
18+
chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-simulator-runtime.sh
19+
$(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
20+
displayName: 'Clean bot'
21+
continueOnError: true
22+
condition: ${{ eq(parameters.platform, 'ios') }}
23+
timeoutInMinutes: 60
1624
1725
- template: provision.yml
1826
parameters:
@@ -44,7 +52,14 @@ steps:
4452
- pwsh: ./build.ps1 -Script eng/devices/${{ parameters.platform }}.cake --target=cg-uitest --project="${{ parameters.path }}" --appproject="${{ parameters.app }}" --device="${{ parameters.device }}" --apiversion="${{ parameters.version }}" --configuration="${{ parameters.configuration }}" --results="$(TestResultsDirectory)" --binlog="$(LogDirectory)" ${{ parameters.cakeArgs }} --verbosity=diagnostic --where="${{ parameters.where }}"
4553
displayName: $(Agent.JobName)
4654
# retryCountOnTaskFailure: 2
47-
55+
56+
- bash: |
57+
zip -9r "$(LogDirectory)/artifacts/CoreSimulatorLog_$(new-guid).zip" "$HOME/Library/Logs/CoreSimulator/"
58+
zip -9r "$(LogDirectory)/DiagnosticReports_$(new-guid).zip" "$HOME/Library/Logs/DiagnosticReports/"
59+
displayName: Zip Simulator Logs
60+
condition: ${{ eq(parameters.platform, 'ios') }}
61+
continueOnError: true
62+
4863
- task: PublishTestResults@2
4964
displayName: Publish the $(System.PhaseName) test results
5065
condition: always()

‎eng/pipelines/common/ui-tests-steps.yml

+16-1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,13 @@ parameters:
1010
configuration : "Release"
1111

1212
steps:
13+
- ${{ if eq(parameters.platform, 'ios')}}:
14+
- bash: |
15+
chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
16+
$(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
17+
displayName: 'Clean bot'
18+
continueOnError: true
19+
timeoutInMinutes: 60
1320
1421
- template: provision.yml
1522
parameters:
@@ -102,7 +109,15 @@ steps:
102109
- pwsh: ./build.ps1 -Script eng/devices/${{ parameters.platform }}.cake --target=uitest --project="${{ parameters.path }}" --appproject="${{ parameters.app }}" --device="${{ parameters.device }}" --apiversion="${{ parameters.version }}" --configuration="${{ parameters.configuration }}" --results="$(TestResultsDirectory)" --binlog="$(LogDirectory)" ${{ parameters.cakeArgs }} --verbosity=diagnostic
103110
displayName: $(Agent.JobName)
104111
retryCountOnTaskFailure: 2
105-
112+
113+
- ${{ if eq(parameters.platform, 'ios')}}:
114+
- bash: |
115+
zip -9r "$(LogDirectory)/CoreSimulatorLog_$(new-guid).zip" "$HOME/Library/Logs/CoreSimulator/"
116+
zip -9r "$(LogDirectory)/DiagnosticReports_$(new-guid).zip" "$HOME/Library/Logs/DiagnosticReports/"
117+
displayName: Zip Simulator Logs
118+
condition: always()
119+
continueOnError: true
120+
106121
- task: PublishTestResults@2
107122
displayName: Publish the $(System.PhaseName) test results
108123
condition: always()

‎eng/scripts/clean-bot.sh

+129
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
#!/bin/bash -ex
2+
3+
# Print disk status before cleaning
4+
df -h
5+
6+
# We don't care about errors in this section, we just want to clean as much as possible
7+
set +e
8+
9+
# Delete all the simulator devices. These can take up a lot of space over time (I've seen 100+GB on the bots)
10+
/Applications/Xcode.app/Contents/Developer/usr/bin/simctl delete all
11+
12+
# Delete old Xcodes.
13+
ls -lad /Applications/Xcode*.app
14+
15+
oldXcodes=(
16+
"/Applications/Xcode44.app"
17+
"/Applications/Xcode5.app"
18+
"/Applications/Xcode502.app"
19+
"/Applications/Xcode511.app"
20+
"/Applications/Xcode6.0.1.app"
21+
"/Applications/Xcode6.app"
22+
"/Applications/Xcode601.app"
23+
"/Applications/Xcode61.app"
24+
"/Applications/Xcode611.app"
25+
"/Applications/Xcode62.app"
26+
"/Applications/Xcode63.app"
27+
"/Applications/Xcode64.app"
28+
"/Applications/Xcode7.app"
29+
"/Applications/Xcode701.app"
30+
"/Applications/Xcode71.app"
31+
"/Applications/Xcode711.app"
32+
"/Applications/Xcode72.app"
33+
"/Applications/Xcode731.app"
34+
"/Applications/Xcode8-GM.app"
35+
"/Applications/Xcode8.app"
36+
"/Applications/Xcode81-GM.app"
37+
"/Applications/Xcode81.app"
38+
"/Applications/Xcode82.app"
39+
"/Applications/Xcode821.app"
40+
"/Applications/Xcode83.app"
41+
"/Applications/Xcode833.app"
42+
"/Applications/Xcode9-GM.app"
43+
"/Applications/Xcode9.app"
44+
"/Applications/Xcode91.app"
45+
"/Applications/Xcode92.app"
46+
"/Applications/Xcode93.app"
47+
"/Applications/Xcode94.app"
48+
"/Applications/Xcode941.app"
49+
"/Applications/Xcode10.app"
50+
"/Applications/Xcode101-beta2.app"
51+
"/Applications/Xcode101-beta3.app"
52+
"/Applications/Xcode101.app"
53+
"/Applications/Xcode102-beta1.app"
54+
"/Applications/Xcode102.app"
55+
"/Applications/Xcode1021.app"
56+
"/Applications/Xcode103.app"
57+
"/Applications/Xcode10GM.app"
58+
"/Applications/Xcode11-beta3.app"
59+
"/Applications/Xcode11-GM.app"
60+
"/Applications/Xcode11.app"
61+
"/Applications/Xcode111.app"
62+
"/Applications/Xcode112.app"
63+
"/Applications/Xcode1121.app"
64+
"/Applications/Xcode113.app"
65+
"/Applications/Xcode1131.app"
66+
"/Applications/Xcode114-beta1.app"
67+
"/Applications/Xcode114-beta2.app"
68+
"/Applications/Xcode114-beta3.app"
69+
"/Applications/Xcode114.app"
70+
"/Applications/Xcode1141.app"
71+
"/Applications/Xcode115-beta1.app"
72+
"/Applications/Xcode115-beta2.app"
73+
"/Applications/Xcode115-GM.app"
74+
"/Applications/Xcode_8.0.app"
75+
"/Applications/Xcode_8.1.app"
76+
"/Applications/Xcode_8.2.1.app"
77+
"/Applications/Xcode_8.3.3.app"
78+
"/Applications/Xcode_9.0.app"
79+
"/Applications/Xcode_9.1.0.app"
80+
"/Applications/Xcode_9.2.0.app"
81+
"/Applications/Xcode_9.2.app"
82+
"/Applications/Xcode_9.4.1.app"
83+
# Xcode 10.2.1 is currently used by Binding Tools for Swift # /Applications/Xcode_10.2.1.app
84+
"/Applications/Xcode_11.3.0.app"
85+
"/Applications/Xcode_11.5.0.app"
86+
"/Applications/Xcode_11.6.0-beta1.app"
87+
"/Applications/Xcode_12.0.0-beta1.app"
88+
"/Applications/Xcode_12.0.0-beta2.app"
89+
"/Applications/Xcode_12.0.0-beta3.app"
90+
"/Applications/Xcode_12.0.0-beta4.app"
91+
"/Applications/Xcode_12.0.0-beta5.app"
92+
"/Applications/Xcode_12.0.0-beta6.app"
93+
"/Applications/Xcode_12.1.0-GM.app"
94+
"/Applications/Xcode_12.0.0-GMb.app"
95+
"/Applications/Xcode_12.2.0-beta1.app"
96+
"/Applications/Xcode_12.2.0-beta2.app"
97+
"/Applications/Xcode_12.2.0-beta3.app"
98+
"/Applications/Xcode_12.2.0-beta.3.app"
99+
"/Applications/Xcode_12.2.0-rc.app"
100+
"/Applications/Xcode_12.5.0-rc.app"
101+
"/Applications/Xcode_13.0.0-beta.app"
102+
"/Applications/Xcode_13.0.0-beta2.app"
103+
"/Applications/Xcode_13.0.0-beta3.app"
104+
"/Applications/Xcode_14.3.1.app"
105+
"/Applications/Xcode_15.0.0.app"
106+
"/Applications/Xcode_15.0.1.app"
107+
"/Applications/Xcode_15.1.0.app"
108+
)
109+
110+
# remove wrongly added .xip files under /Applications, confuses provisionator and
111+
# are not needed and wrong
112+
sudo rm -Rf /Applications/*.xip
113+
114+
# pick the current selected xcode to make sure we do not remove it.
115+
XCODE_SELECT=$(xcode-select -p)
116+
117+
for oldXcode in "${oldXcodes[@]}"; do
118+
if [ "$XCODE_SELECT" != "$oldXcode/Contents/Developer" ]; then
119+
sudo rm -Rf "$oldXcode"
120+
else
121+
echo "Not removing $oldXcode because is the currently selected one."
122+
fi
123+
done
124+
125+
DIR="$(dirname "${BASH_SOURCE[0]}")"
126+
"$DIR"/clean-simulator-runtime.sh
127+
128+
# Print disk status after cleaning
129+
df -h

‎eng/scripts/clean-simulator-runtime.sh

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/bin/bash -eux
2+
3+
# find if there are any duplicated simulator runtimes for a given platform
4+
5+
set -o pipefail
6+
IFS=$'\n\t'
7+
8+
xcrun simctl runtime list -j > simruntime.json
9+
cat simruntime.json
10+
11+
grep -e '"identifier" : ' -e '"runtimeIdentifier" : ' simruntime.json | tr '\n' ' ' | sed -e 's/,//g' -e 's/"//g' -e 's/runtimeIdentifier : //g' -e $'s/identifier : /@/g' | tr '@' '\n' | awk NF | sed 's/^[[:blank:]]*//' > simruntime-lines.txt
12+
cat simruntime-lines.txt
13+
14+
sed -e 's/.*com.apple/com.apple/g' simruntime-lines.txt > simruntime-runtimes.txt
15+
cat simruntime-runtimes.txt
16+
17+
sort simruntime-runtimes.txt | uniq -c | sort -n | sed 's/^[[:blank:]]*//' > simruntime-runtimes-by-count.txt
18+
cat simruntime-runtimes-by-count.txt
19+
20+
grep -v '^1 ' simruntime-runtimes-by-count.txt | sed 's/^[0-9 ]*//' > simruntime-duplicated-runtimes.txt
21+
cat simruntime-duplicated-runtimes.txt
22+
23+
while IFS= read -r simruntime
24+
do
25+
echo "Duplicated: $simruntime"
26+
grep "$simruntime" simruntime-lines.txt | sed 's/ .*//' | while IFS= read -r id
27+
do
28+
echo " sudo xcrun simctl runtime delete $id"
29+
if ! sudo xcrun simctl runtime delete "$id"; then
30+
echo " failed to delete runtime $id"
31+
else
32+
echo " deleted runtime $id"
33+
fi
34+
done
35+
done < simruntime-duplicated-runtimes.txt
36+
37+
xcrun simctl runtime list -v

0 commit comments

Comments
 (0)