Skip to content

Commit dc8265b

Browse files
FizzadarFizzadar
authored
Complete files quote migration (#611)
* Update `make_formatted_string_command` to require explicit quoting. This function now requires arguments to be explicitly quoted. The change makes it possible to quote some but not all arguments. It also removes the magic and makes its usage explicit. * Update files operations to explicitly quote formatting values. * Add new `files.*` operation tests for filenames that require quotes. * Update `files.*` facts to properly quote filenames. * Add `__repr__` for `QuoteString` class. * Remove pointless non-idempotent touch flag on `files.file` tests. * Correct docstring on `make_formatted_string_command`. * Properly escape paths for regex in file hash facts. * Add tests for files facts with paths that need quoting.
1 parent 7a2bc1c commit dc8265b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+268
-122
lines changed

‎pyinfra/api/command.py

Lines changed: 22 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,41 @@
11
import shlex
22

3+
from string import Formatter
4+
35
from six.moves import shlex_quote
46

57
from .operation_kwargs import get_executor_kwarg_keys
68

79

810
def make_formatted_string_command(string, *args, **kwargs):
911
'''
10-
Helper function that takes a shell command or script as a string, splits
11-
it, applies formatting to each bit - quoting any formatted values - before
12-
returning them as a `StringCommand` object.
12+
Helper function that takes a shell command or script as a string, splits it
13+
using ``shlex.split`` and then formats each bit, returning a ``StringCommand``
14+
instance with each bit.
1315
1416
Useful to enable string formatted commands/scripts, for example:
1517
1618
.. code:: python
1719
18-
curl_command = make_formatted_string_command('curl -sSLf {0} -o {1}', src, dest)
20+
curl_command = make_formatted_string_command(
21+
'curl -sSLf {0} -o {1}',
22+
QuoteString(src),
23+
QuoteString(dest),
24+
)
1925
'''
2026

21-
formatted_bits = []
27+
formatter = Formatter()
28+
string_bits = []
2229

2330
for bit in shlex.split(string):
24-
formatted = bit.format(*args, **kwargs)
25-
if bit.startswith('{') and bit.endswith('}'):
26-
formatted_bits.append(QuoteString(formatted))
27-
else:
28-
formatted_bits.append(formatted)
31+
for item in formatter.parse(bit):
32+
if item[0]:
33+
string_bits.append(item[0])
34+
if item[1]:
35+
value, _ = formatter.get_field(item[1], args, kwargs)
36+
string_bits.append(value)
2937

30-
return StringCommand(*formatted_bits)
38+
return StringCommand(*string_bits)
3139

3240

3341
class MaskString(str):
@@ -38,6 +46,9 @@ class QuoteString(object):
3846
def __init__(self, obj):
3947
self.object = obj
4048

49+
def __repr__(self):
50+
return 'QuoteString({0})'.format(self.object)
51+
4152

4253
class PyinfraCommand(object):
4354
def __init__(self, *args, **kwargs):

‎pyinfra/facts/files.py

Lines changed: 39 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,7 @@
44

55
from datetime import datetime
66

7-
from six.moves import shlex_quote
8-
9-
from pyinfra.api.connectors.util import escape_unix_path
7+
from pyinfra.api.command import make_formatted_string_command, QuoteString
108
from pyinfra.api.facts import FactBase
119
from pyinfra.api.util import try_int
1210

@@ -66,12 +64,11 @@ class File(FactBase):
6664
test_flag = '-e'
6765

6866
def command(self, path):
69-
path = escape_unix_path(path)
70-
return (
71-
'! test {test_flag} {path} || ' # only stat if the file exists
72-
'({linux_stat_command} {path} 2> /dev/null || {bsd_stat_command} {path})'
73-
).format(
74-
path=path,
67+
return make_formatted_string_command((
68+
'! test {test_flag} {0} || ' # only stat if the file exists
69+
'( {linux_stat_command} {0} 2> /dev/null || {bsd_stat_command} {0} )'
70+
),
71+
QuoteString(path),
7572
test_flag=self.test_flag,
7673
linux_stat_command=LINUX_STAT_COMMAND,
7774
bsd_stat_command=BSD_STAT_COMMAND,
@@ -143,17 +140,16 @@ class Sha1File(FactBase):
143140
]
144141

145142
def command(self, name):
146-
name = escape_unix_path(name)
147143
self.name = name
148-
return (
149-
'test -e {0} && ('
150-
'sha1sum {0} 2> /dev/null || shasum {0} 2> /dev/null || sha1 {0}'
144+
return make_formatted_string_command((
145+
'test -e {0} && ( '
146+
'sha1sum {0} 2> /dev/null || shasum {0} 2> /dev/null || sha1 {0} '
151147
') || true'
152-
).format(name)
148+
), QuoteString(name))
153149

154150
def process(self, output):
155151
for regex in self._regexes:
156-
regex = regex % self.name
152+
regex = regex % re.escape(self.name)
157153
matches = re.match(regex, output[0])
158154
if matches:
159155
return matches.group(1)
@@ -170,19 +166,18 @@ class Sha256File(FactBase):
170166
]
171167

172168
def command(self, name):
173-
name = escape_unix_path(name)
174169
self.name = name
175-
return (
176-
'test -e {0} && ('
170+
return make_formatted_string_command((
171+
'test -e {0} && ( '
177172
'sha256sum {0} 2> /dev/null '
178173
'|| shasum -a 256 {0} 2> /dev/null '
179-
'|| sha256 {0}'
174+
'|| sha256 {0} '
180175
') || true'
181-
).format(name)
176+
), QuoteString(name))
182177

183178
def process(self, output):
184179
for regex in self._regexes:
185-
regex = regex % self.name
180+
regex = regex % re.escape(self.name)
186181
matches = re.match(regex, output[0])
187182
if matches:
188183
return matches.group(1)
@@ -199,13 +194,15 @@ class Md5File(FactBase):
199194
]
200195

201196
def command(self, name):
202-
name = escape_unix_path(name)
203197
self.name = name
204-
return 'test -e {0} && (md5sum {0} 2> /dev/null || md5 {0}) || true'.format(name)
198+
return make_formatted_string_command(
199+
'test -e {0} && ( md5sum {0} 2> /dev/null || md5 {0} ) || true',
200+
QuoteString(name),
201+
)
205202

206203
def process(self, output):
207204
for regex in self._regexes:
208-
regex = regex % self.name
205+
regex = regex % re.escape(self.name)
209206
matches = re.match(regex, output[0])
210207
if matches:
211208
return matches.group(1)
@@ -218,21 +215,20 @@ class FindInFile(FactBase):
218215
'''
219216

220217
def command(self, name, pattern):
221-
name = escape_unix_path(name)
222-
pattern = shlex_quote(pattern)
218+
self.exists_name = '__pyinfra_exists_{0}'.format(name)
223219

224-
self.name = name
220+
# TODO: remove special charts from __pyinfra_exists thing
225221

226-
return (
222+
return make_formatted_string_command((
227223
'grep -e {0} {1} 2> /dev/null || '
228-
'(find {1} -type f > /dev/null && echo "__pyinfra_exists_{1}" || true)'
229-
).format(pattern, name).strip()
224+
'( find {1} -type f > /dev/null && echo {2} || true )'
225+
), QuoteString(pattern), QuoteString(name), QuoteString(self.exists_name))
230226

231227
def process(self, output):
232228
# If output is the special string: no matches, so return an empty list;
233229
# this allows us to differentiate between no matches in an existing file
234230
# or a file not existing.
235-
if output and output[0] == '__pyinfra_exists_{0}'.format(self.name):
231+
if output and output[0] == self.exists_name:
236232
return []
237233

238234
return output
@@ -245,7 +241,10 @@ class FindFiles(FactBase):
245241

246242
@staticmethod
247243
def command(name):
248-
return 'find {0} -type f || true'.format(escape_unix_path(name))
244+
return make_formatted_string_command(
245+
'find {0} -type f || true',
246+
QuoteString(name),
247+
)
249248

250249
@staticmethod
251250
def process(output):
@@ -259,7 +258,10 @@ class FindLinks(FindFiles):
259258

260259
@staticmethod
261260
def command(name):
262-
return 'find {0} -type l || true'.format(escape_unix_path(name))
261+
return make_formatted_string_command(
262+
'find {0} -type l || true',
263+
QuoteString(name),
264+
)
263265

264266

265267
class FindDirectories(FindFiles):
@@ -269,4 +271,7 @@ class FindDirectories(FindFiles):
269271

270272
@staticmethod
271273
def command(name):
272-
return 'find {0} -type d || true'.format(escape_unix_path(name))
274+
return make_formatted_string_command(
275+
'find {0} -type d || true',
276+
QuoteString(name),
277+
)

‎pyinfra/operations/files.py

Lines changed: 28 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -118,17 +118,23 @@ def download(
118118

119119
# If we download, always do user/group/mode as SSH user may be different
120120
if download:
121-
curl_command = make_formatted_string_command('curl -sSLf {0} -o {1}', src, dest)
121+
curl_command = make_formatted_string_command(
122+
'curl -sSLf {0} -o {1}',
123+
QuoteString(src),
124+
QuoteString(dest),
125+
)
122126
wget_command = make_formatted_string_command(
123-
'wget -q {0} -O {1} || (rm -f {1}; exit 1)', src, dest,
127+
'wget -q {0} -O {1} || ( rm -f {1} ; exit 1 )',
128+
QuoteString(src),
129+
QuoteString(dest),
124130
)
125131

126132
if host.fact.which('curl'):
127133
yield curl_command
128134
elif host.fact.which('wget'):
129135
yield wget_command
130136
else:
131-
yield '({0}) || ({1})'.format(curl_command, wget_command)
137+
yield '( {0} ) || ( {1} )'.format(curl_command, wget_command)
132138

133139
if user or group:
134140
yield chown(dest, user, group)
@@ -138,21 +144,21 @@ def download(
138144

139145
if sha1sum:
140146
yield make_formatted_string_command((
141-
'((sha1sum {0} 2> /dev/null || shasum {0} || sha1 {0}) | grep {1}) '
142-
'|| (echo {2} && exit 1)'
143-
), dest, sha1sum, 'SHA1 did not match!')
147+
'(( sha1sum {0} 2> /dev/null || shasum {0} || sha1 {0} ) | grep {1} ) '
148+
'|| ( echo {2} && exit 1 )'
149+
), QuoteString(dest), sha1sum, QuoteString('SHA1 did not match!'))
144150

145151
if sha256sum:
146152
yield make_formatted_string_command((
147-
'((sha256sum {0} 2> /dev/null || shasum -a 256 {0} || sha256 {0}) | grep {1}) '
148-
'|| (echo {2} && exit 1)'
149-
), dest, sha256sum, 'SHA256 did not match!')
153+
'(( sha256sum {0} 2> /dev/null || shasum -a 256 {0} || sha256 {0} ) | grep {1}) '
154+
'|| ( echo {2} && exit 1 )'
155+
), QuoteString(dest), sha256sum, QuoteString('SHA256 did not match!'))
150156

151157
if md5sum:
152158
yield make_formatted_string_command((
153-
'((md5sum {0} 2> /dev/null || md5 {0}) | grep {1}) '
154-
'|| (echo {2} && exit 1)'
155-
), dest, md5sum, 'MD5 did not match!')
159+
'(( md5sum {0} 2> /dev/null || md5 {0} ) | grep {1}) '
160+
'|| ( echo {2} && exit 1 )'
161+
), QuoteString(dest), md5sum, QuoteString('MD5 did not match!'))
156162

157163
else:
158164
host.noop('file {0} has already been downloaded'.format(dest))
@@ -259,14 +265,16 @@ def line(
259265
replace = ''
260266

261267
# Save commands for re-use in dynamic script when file not present at fact stage
262-
echo_command_formatter = 'echo "{0}" >> {1}' if interpolate_variables else "echo '{0}' >> {1}"
263268
echo_command = make_formatted_string_command(
264-
echo_command_formatter, line, path,
269+
'echo {0} >> {1}',
270+
'"{0}"'.format(line) if interpolate_variables else QuoteString(line),
271+
QuoteString(path),
265272
)
266273

267274
if backup:
275+
backup_filename = '{0}.{1}'.format(path, get_timestamp())
268276
echo_command = StringCommand(make_formatted_string_command(
269-
'cp {0} {0}.{1} && ', path, get_timestamp(),
277+
'cp {0} {1} && ', QuoteString(path), QuoteString(backup_filename),
270278
), echo_command)
271279

272280
sed_replace_command = sed_replace(
@@ -284,15 +292,15 @@ def line(
284292
yield make_formatted_string_command(
285293
'''
286294
if [ -f '{target}' ]; then
287-
(grep {match_line} '{target}' && \
295+
( grep {match_line} '{target}' && \
288296
{sed_replace_command}) 2> /dev/null || \
289-
{echo_command};
297+
{echo_command} ;
290298
else
291-
{echo_command};
299+
{echo_command} ;
292300
fi
293301
''',
294-
target=path,
295-
match_line=match_line,
302+
target=QuoteString(path),
303+
match_line=QuoteString(match_line),
296304
echo_command=echo_command,
297305
sed_replace_command=sed_replace_command,
298306
)

‎tests/facts/directory/file.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"arg": "/path/to/a/file",
3-
"command": "! test -e /path/to/a/file || (stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /path/to/a/file 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /path/to/a/file)",
3+
"command": "! test -e /path/to/a/file || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /path/to/a/file 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /path/to/a/file )",
44
"output": [
55
"user=pyinfra group=pyinfra mode=-rwxrwxrwx atime=1594804767 mtime=1594804767 ctime=0 size=8 '/path/to/a/file'"
66
],

‎tests/facts/directory/link.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"arg": "/home/pyinfra/mylink",
3-
"command": "! test -e /home/pyinfra/mylink || (stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra/mylink 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra/mylink)",
3+
"command": "! test -e /home/pyinfra/mylink || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra/mylink 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra/mylink )",
44
"output": [
55
"user=root group=root mode=lrwxrwxrwx atime=1594804774 mtime=1594804770 ctime=0 size=6 '/home/pyinfra/mylink' -> 'file.txt'"
66
],

‎tests/facts/directory/valid.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"arg": "/home/pyinfra/myd@-_ir",
3-
"command": "! test -e /home/pyinfra/myd@-_ir || (stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra/myd@-_ir 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra/myd@-_ir)",
3+
"command": "! test -e /home/pyinfra/myd@-_ir || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra/myd@-_ir 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra/myd@-_ir )",
44
"output": [
55
"user=pyinfra group=pyinfra mode=drw-r--r-- atime=1594804583 mtime=1594804583 ctime=0 size=0 '/home/pyinfra/myd@-_ir'"
66
],

‎tests/facts/file/directory.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"arg": "/home/pyinfra",
3-
"command": "! test -e /home/pyinfra || (stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra)",
3+
"command": "! test -e /home/pyinfra || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra )",
44
"output": [
55
"user=root group=root mode=drw-r--r-- atime=1594804583 mtime=1594804583 ctime=0 size=0 '/home/pyinfra'"
66
],

‎tests/facts/file/invalid_output.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"arg": "/home/pyinfra/fil-@_e.txt",
3-
"command": "! test -e /home/pyinfra/fil-@_e.txt || (stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra/fil-@_e.txt 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra/fil-@_e.txt)",
3+
"command": "! test -e /home/pyinfra/fil-@_e.txt || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra/fil-@_e.txt 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra/fil-@_e.txt )",
44
"output": [
55
"not-gonna-match"
66
],

‎tests/facts/file/link.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"arg": "/home/pyinfra/mylink",
3-
"command": "! test -e /home/pyinfra/mylink || (stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra/mylink 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra/mylink)",
3+
"command": "! test -e /home/pyinfra/mylink || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra/mylink 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra/mylink )",
44
"output": [
55
"user=root group=root mode=lrwxrwxrwx atime=1594804774 mtime=1594804770 ctime=0 size=6 '/home/pyinfra/mylink' -> 'file.txt'"
66
],

‎tests/facts/file/valid.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"arg": "/home/pyinfra/fil-@_e.txt",
3-
"command": "! test -e /home/pyinfra/fil-@_e.txt || (stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra/fil-@_e.txt 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra/fil-@_e.txt)",
3+
"command": "! test -e /home/pyinfra/fil-@_e.txt || ( stat -c 'user=%U group=%G mode=%A atime=%X mtime=%Y ctime=%Z size=%s %N' /home/pyinfra/fil-@_e.txt 2> /dev/null || stat -f 'user=%Su group=%Sg mode=%Sp atime=%a mtime=%m ctime=%c size=%z %N%SY' /home/pyinfra/fil-@_e.txt )",
44
"output": [
55
"user=pyinfra group=domain users mode=-rwxrwx--- atime=1594804767 mtime=1594804767 ctime=0 size=8 '/home/pyinfra/fil-@_e.txt'"
66
],

0 commit comments

Comments
 (0)