Skip to content

Commit e4a43bf

Browse files
dflookdflook
authored
Merge pull request #238 from dflook/unlock-state
Unlock state
2 parents d37748b + a5ddd1b commit e4a43bf

File tree

13 files changed

+642
-8
lines changed

13 files changed

+642
-8
lines changed

‎.github/workflows/test-unlock-state.yaml‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
name: Test terraform-unlock-state
2+
3+
on:
4+
- pull_request
5+
6+
env:
7+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
8+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
9+
10+
jobs:
11+
default_workspace:
12+
runs-on: ubuntu-latest
13+
name: Default workspace
14+
steps:
15+
- name: Checkout
16+
uses: actions/checkout@v3
17+
18+
- name: Check state is not locked
19+
uses: ./terraform-apply
20+
with:
21+
path: tests/workflows/test-unlock-state
22+
auto_approve: true
23+
24+
- name: Intentionally lock the state
25+
uses: ./terraform-apply
26+
id: failed-apply
27+
continue-on-error: true
28+
with:
29+
path: tests/workflows/test-unlock-state
30+
auto_approve: true
31+
variables:
32+
lock=true
33+
34+
# State is now locked
35+
36+
- name: Check apply-failed
37+
run: |
38+
if [[ "${{ steps.failed-apply.outcome }}" != "failure" ]]; then
39+
echo "Apply did not fail correctly"
40+
exit 1
41+
fi
42+
43+
if [[ "${{ steps.failed-apply.outputs.failure-reason }}" != "apply-failed" ]]; then
44+
echo "::error:: failure-reason not set correctly"
45+
exit 1
46+
fi
47+
48+
# Check state-locked
49+
- name: Try using locked state
50+
uses: ./terraform-apply
51+
id: locked-state
52+
continue-on-error: true
53+
with:
54+
path: tests/workflows/test-unlock-state
55+
auto_approve: true
56+
57+
- name: Check state locked failure-reason
58+
run: |
59+
if [[ "${{ steps.locked-state.outcome }}" != "failure" ]]; then
60+
echo "Apply did not fail correctly"
61+
exit 1
62+
fi
63+
64+
if [[ "${{ steps.locked-state.outputs.failure-reason }}" != "state-locked" ]]; then
65+
echo "::error:: failure-reason not set correctly"
66+
exit 1
67+
fi
68+
69+
echo '"${{ steps.locked-state.outputs.lock-info }}"'
70+
71+
echo 'Lock id is ${{ fromJson(steps.locked-state.outputs.lock-info).ID }}'
72+
73+
- name: Unlock the state
74+
uses: ./terraform-unlock-state
75+
continue-on-error: true
76+
with:
77+
path: tests/workflows/test-unlock-state
78+
lock_id: ${{ fromJson(steps.locked-state.outputs.lock-info).ID }}
79+
80+
- name: Check state is not locked
81+
uses: ./terraform-apply
82+
with:
83+
path: tests/workflows/test-unlock-state
84+
auto_approve: true
85+
86+
nondefault_workspace:
87+
runs-on: ubuntu-latest
88+
name: Non Default workspace
89+
steps:
90+
- name: Checkout
91+
uses: actions/checkout@v3
92+
93+
- name: Create first workspace
94+
uses: ./terraform-new-workspace
95+
with:
96+
path: tests/workflows/test-unlock-state
97+
workspace: hello
98+
99+
- name: Check state is not locked
100+
uses: ./terraform-apply
101+
with:
102+
path: tests/workflows/test-unlock-state
103+
workspace: hello
104+
auto_approve: true
105+
106+
- name: Intentionally lock the state
107+
uses: ./terraform-apply
108+
id: failed-apply-workspace
109+
continue-on-error: true
110+
with:
111+
path: tests/workflows/test-unlock-state
112+
workspace: hello
113+
auto_approve: true
114+
variables:
115+
lock=true
116+
117+
# State is now locked
118+
119+
- name: Check apply-failed
120+
run: |
121+
if [[ "${{ steps.failed-apply-workspace.outcome }}" != "failure" ]]; then
122+
echo "Apply did not fail correctly"
123+
exit 1
124+
fi
125+
126+
if [[ "${{ steps.failed-apply-workspace.outputs.failure-reason }}" != "apply-failed" ]]; then
127+
echo "::error:: failure-reason not set correctly"
128+
exit 1
129+
fi
130+
131+
# Check state-locked
132+
- name: Try using locked state
133+
uses: ./terraform-apply
134+
id: locked-state-workspace
135+
continue-on-error: true
136+
with:
137+
path: tests/workflows/test-unlock-state
138+
workspace: hello
139+
auto_approve: true
140+
141+
- name: Check state locked failure-reason
142+
run: |
143+
if [[ "${{ steps.locked-state-workspace.outcome }}" != "failure" ]]; then
144+
echo "Apply did not fail correctly"
145+
exit 1
146+
fi
147+
148+
if [[ "${{ steps.locked-state-workspace.outputs.failure-reason }}" != "state-locked" ]]; then
149+
echo "::error:: failure-reason not set correctly"
150+
exit 1
151+
fi
152+
153+
echo '"${{ steps.locked-state-workspace.outputs.lock-info }}"'
154+
155+
echo 'Lock id is ${{ fromJson(steps.locked-state-workspace.outputs.lock-info).ID }}'
156+
157+
- name: Unlock the state
158+
uses: ./terraform-unlock-state
159+
continue-on-error: true
160+
with:
161+
path: tests/workflows/test-unlock-state
162+
workspace: hello
163+
lock_id: ${{ fromJson(steps.locked-state-workspace.outputs.lock-info).ID }}
164+
165+
- name: Check state is not locked
166+
uses: ./terraform-apply
167+
with:
168+
path: tests/workflows/test-unlock-state
169+
workspace: hello
170+
auto_approve: true
171+
172+
- name: Destroy workspace
173+
uses: ./terraform-destroy-workspace
174+
with:
175+
path: tests/workflows/test-unlock-state
176+
workspace: hello

‎image/actions.sh‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -399,11 +399,11 @@ function plan() {
399399

400400
function destroy() {
401401
# shellcheck disable=SC2086
402-
debug_log terraform destroy -input=false -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS
402+
debug_log terraform destroy -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS
403403

404404
set +e
405405
# shellcheck disable=SC2086
406-
(cd "$INPUT_PATH" && terraform destroy -input=false -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS) \
406+
(cd "$INPUT_PATH" && terraform destroy -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS) \
407407
2>"$STEP_TMP_DIR/terraform_destroy.stderr" \
408408
| tee /dev/fd/3 \
409409
>"$STEP_TMP_DIR/terraform_destroy.stdout"
@@ -413,6 +413,12 @@ function destroy() {
413413
set -e
414414
}
415415

416+
function force_unlock() {
417+
echo "Unlocking state with ID: $INPUT_LOCK_ID"
418+
debug_log terraform force-unlock -force $INPUT_LOCK_ID
419+
(cd "$INPUT_PATH" && terraform force-unlock -force $INPUT_LOCK_ID)
420+
}
421+
416422
# Every file written to disk should use one of these directories
417423
STEP_TMP_DIR="/tmp"
418424
JOB_TMP_DIR="$HOME/.dflook-terraform-github-actions"

‎image/entrypoints/apply.sh‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,24 @@ function apply() {
2424
# shellcheck disable=SC2086
2525
debug_log terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_OUT
2626
# shellcheck disable=SC2086
27-
(cd "$INPUT_PATH" && terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_OUT) | $TFMASK
27+
(cd "$INPUT_PATH" && terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_OUT) \
28+
2>"$STEP_TMP_DIR/terraform_apply.stderr" \
29+
| $TFMASK
2830
APPLY_EXIT=${PIPESTATUS[0]}
31+
>&2 cat "$STEP_TMP_DIR/terraform_apply.stderr"
2932
else
3033
# There is no plan file to apply, since the remote backend can't produce them.
3134
# Instead we need to do an auto approved apply using the arguments we would normally use for the plan
3235

3336
# shellcheck disable=SC2086
3437
debug_log terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG '$PLAN_ARGS' # don't expand plan args
3538
# shellcheck disable=SC2086
36-
(cd "$INPUT_PATH" && terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS) | $TFMASK | tee "$STEP_TMP_DIR/terraform_apply.stdout"
39+
(cd "$INPUT_PATH" && terraform apply -input=false -no-color -auto-approve -lock-timeout=300s $PARALLEL_ARG $PLAN_ARGS) \
40+
2>"$STEP_TMP_DIR/terraform_apply.stderr" \
41+
| $TFMASK \
42+
| tee "$STEP_TMP_DIR/terraform_apply.stdout"
3743
APPLY_EXIT=${PIPESTATUS[0]}
44+
>&2 cat "$STEP_TMP_DIR/terraform_apply.stderr"
3845

3946
if remote-run-id "$STEP_TMP_DIR/terraform_apply.stdout" >"$STEP_TMP_DIR/remote-run-id.stdout" 2>"$STEP_TMP_DIR/remote-run-id.stderr"; then
4047
RUN_ID="$(<"$STEP_TMP_DIR/remote-run-id.stdout")"
@@ -49,7 +56,11 @@ function apply() {
4956
if [[ $APPLY_EXIT -eq 0 ]]; then
5057
update_status ":white_check_mark: Plan applied in $(job_markdown_ref)"
5158
else
52-
set_output failure-reason apply-failed
59+
if lock-info "$STEP_TMP_DIR/terraform_apply.stderr"; then
60+
set_output failure-reason state-locked
61+
else
62+
set_output failure-reason apply-failed
63+
fi
5364
update_status ":x: Error applying plan in $(job_markdown_ref)"
5465
exit 1
5566
fi
@@ -76,6 +87,10 @@ fi
7687
if [[ $PLAN_EXIT -eq 1 ]]; then
7788
cat >&2 "$STEP_TMP_DIR/terraform_plan.stderr"
7889

90+
if lock-info "$STEP_TMP_DIR/terraform_plan.stderr"; then
91+
set_output failure-reason state-locked
92+
fi
93+
7994
update_status ":x: Error applying plan in $(job_markdown_ref)"
8095
exit 1
8196
fi

‎image/entrypoints/destroy-workspace.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,11 @@ fi
2121

2222
if [[ $DESTROY_EXIT -eq 1 ]]; then
2323
cat >&2 "$STEP_TMP_DIR/terraform_destroy.stderr"
24-
set_output failure-reason destroy-failed
24+
if lock-info "$STEP_TMP_DIR/terraform_destroy.stderr"; then
25+
set_output failure-reason state-locked
26+
else
27+
set_output failure-reason destroy-failed
28+
fi
2529
exit 1
2630
fi
2731

‎image/entrypoints/destroy.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@ fi
2121

2222
if [[ $DESTROY_EXIT -eq 1 ]]; then
2323
cat >&2 "$STEP_TMP_DIR/terraform_destroy.stderr"
24-
set_output failure-reason destroy-failed
24+
if lock-info "$STEP_TMP_DIR/terraform_destroy.stderr"; then
25+
set_output failure-reason state-locked
26+
else
27+
set_output failure-reason destroy-failed
28+
fi
2529
exit 1
2630
fi

‎image/entrypoints/unlock-state.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
3+
# shellcheck source=../actions.sh
4+
source /usr/local/actions.sh
5+
6+
debug
7+
setup
8+
init-backend-workspace
9+
10+
force_unlock

‎image/setup.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@
1515
'plan_summary=plan_summary.__main__:main',
1616
'terraform-cloud-state=terraform_cloud_state.__main__:main',
1717
'remote-run-id=terraform_cloud_state.__main__:remote_run_id',
18-
'get-terraform-checksums=terraform_version.get_checksums:main'
18+
'get-terraform-checksums=terraform_version.get_checksums:main',
19+
'lock-info=lock_info.__main__:main'
1920
]
2021
},
2122
install_requires=[

‎image/src/lock_info/__init__.py‎

Whitespace-only changes.

‎image/src/lock_info/__main__.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
"""
2+
Create an actions output for any lock info
3+
4+
Input arg is a path to a file containing stderr from a terraform command
5+
If any state lock info is found creates a `lock-info` actions output with a json dump of the lock info
6+
and has a zero exit code.
7+
8+
If no state locked error is present, has a nonzero exit
9+
10+
Usage:
11+
lock-info <TERRAFORM_STDERR_PATH>
12+
"""
13+
14+
import json
15+
import re
16+
import sys
17+
from typing import Iterable, Optional
18+
from github_actions.commands import output
19+
20+
def get_lock_info(stderr: Iterable[str]) -> Optional[dict[str, str]]:
21+
locked = False
22+
lock_info_line = False
23+
lock_info = {}
24+
25+
for line in stderr:
26+
if locked is True:
27+
if lock_info_line:
28+
if match := re.match(r'^\s+(?P<field>.*?):\s+(?P<value>.*)', line):
29+
lock_info[match['field']] = match['value']
30+
31+
elif line.startswith('Lock Info:'):
32+
lock_info_line = True
33+
34+
elif 'Error acquiring the state lock' in line:
35+
locked = True
36+
37+
return lock_info if locked else None
38+
39+
40+
def main():
41+
with open(sys.argv[1]) as f:
42+
lock_info = get_lock_info(f.readlines())
43+
44+
if lock_info is None:
45+
sys.exit(1)
46+
47+
output('lock-info', json.dumps(lock_info))
48+
49+
50+
if __name__ == '__main__':
51+
main()

0 commit comments

Comments
 (0)