Skip to content

Commit 432e8c5

Browse files
ashishbijlaniashishbijlani
committed
Fix errors during pypi package audit
Signed-off-by: Ashish Bijlani <ashish.bijlani@gmail.com>
1 parent e3dc925 commit 432e8c5

File tree

4 files changed

+58
-61
lines changed

4 files changed

+58
-61
lines changed

‎packj/audit/main.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -797,7 +797,6 @@ def parse_request_args(args):
797797
if args.debug:
798798
try:
799799
_, filename = tempfile.mkstemp(prefix='debug_', dir=report_dir, suffix='.log')
800-
os.chmod(filename, 0o544)
801800
print(f'\n*** NOTE: Running in debug mode (log: {filename}) ***\n')
802801
logging.basicConfig(filename=filename, datefmt='%H:%M:%S', level=logging.DEBUG,
803802
format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s')

‎packj/audit/static_proxy/astgen_py.py‎

Lines changed: 26 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -43,26 +43,26 @@ def generic_visit(self, node):
4343
ast.NodeVisitor.generic_visit(self, node)
4444
if self.debug:
4545
if hasattr(node, 'lineno'):
46-
logging.warning('visiting %s node at line %d' % (type(node).__name__, node.lineno))
46+
logging.warning(f'visiting {type(node).__name__} node at line {node.lineno}')
4747
else:
48-
logging.warning('visiting %s node' % (type(node).__name__))
48+
logging.warning(f'visiting {type(node).__name__} node')
4949

5050
def visit_ImportFrom(self, node):
51-
logging.debug('visiting ImportFrom node (line %d)' % (node.lineno))
51+
logging.debug(f'visiting ImportFrom node (line {node.lineno})')
5252
for name in node.names:
5353
self.name2module.setdefault(name.name, node.module)
5454
if name.asname is not None:
5555
self.alias2name.setdefault(name.asname, name.name)
5656
ast.NodeVisitor.generic_visit(self, node)
5757

5858
def visit_FunctionDef(self, node):
59-
logging.debug('visiting FunctionDef node (line %d)' % (node.lineno))
59+
logging.debug(f'visiting FunctionDef node (line {node.lineno})')
6060
# FIXME: warn about redefined functions?
6161
if node.name in self.alias2name or node.name in self.name2module:
62-
logging.warning("redefined imported function %s!" % (node.name))
62+
logging.warning(f'redefined imported function {node.name}!')
6363
ast.NodeVisitor.generic_visit(self, node)
6464
if self.save_feature:
65-
logging.warning("set root_nodes")
65+
logging.warning('set root_nodes')
6666
node_details = {
6767
"Name" : node.name,
6868
"File" : self.infile,
@@ -71,10 +71,10 @@ def visit_FunctionDef(self, node):
7171
self.all_declrefs["Functions"].append(node_details)
7272

7373
def visit_ClassDef(self, node):
74-
logging.debug('visiting ClassDef node (line %d)' % (node.lineno))
74+
logging.debug(f'visiting ClassDef node (line {node.lineno})')
7575
ast.NodeVisitor.generic_visit(self, node)
7676
if self.save_feature:
77-
logging.warning("set root_nodes")
77+
logging.warning('set root_nodes')
7878

7979
node_details = {
8080
"Name" : node.name,
@@ -84,17 +84,17 @@ def visit_ClassDef(self, node):
8484
self.all_declrefs["Classes"].append(node_details)
8585

8686
def visit_Call(self, node):
87-
logging.debug('visiting Call node (line %d)' % (node.lineno))
87+
logging.debug(f'visiting Call node (line {node.lineno})')
8888

8989
# debug code
9090
if self.debug:
9191
for fieldname, value in ast.iter_fields(node):
92-
logging.warning('fieldname %s, value %s' % (fieldname, value))
92+
logging.warning(f'fieldname {fieldname}, value {value}')
9393
if fieldname == 'func':
9494
for f_fieldname, f_value in ast.iter_fields(value):
95-
logging.info('func fieldname %s, func value %s' % (f_fieldname, f_value))
95+
logging.info(f'func fieldname {f_fieldname}, func value {f_value}')
9696
if f_fieldname == 'id':
97-
logging.warning('func id: %s' % (f_value))
97+
logging.warning(f'func id: {f_value}')
9898

9999
# compute base and func
100100
if isinstance(node.func, ast.Attribute):
@@ -104,24 +104,23 @@ def visit_Call(self, node):
104104
base = node.func.value.id
105105
elif isinstance(node.func.value, ast.Call):
106106
base = self.asttok.get_text(node.func.value)
107-
logging.debug("node.func.value is ast.Call, Ignoring!")
107+
logging.debug('node.func.value is ast.Call, Ignoring!')
108108
elif isinstance(node.func.value, ast.Subscript):
109109
base = self.asttok.get_text(node.func.value)
110110
# NOTE: currently, we use text of chained functions (i.e. foo().bar(), foo() is used),
111111
# because Python is runtime type language, and it is not possible to get the type statically
112-
logging.warning("node.func.value type ast.Subscript, fields: %s",
113-
list(ast.iter_fields(node.func.value)))
112+
logging.warning(f'node.func.value type ast.Subscript, fields: {list(ast.iter_fields(node.func.value))}')
114113
else:
115114
base = self.asttok.get_text(node.func.value)
116-
logging.warning("node.func.value type: %s, fields: %s",
117-
type(node.func.value), list(ast.iter_fields(node.func.value)))
115+
logging.warning(f'node.func.value type: {type(node.func.value)}, \
116+
fields: {list(ast.iter_fields(node.func.value))}')
118117
else:
119118
# NOTE: we assume the imported functions are not redefined! this may not be true!
120119
if isinstance(node.func, ast.Name):
121120
name = node.func.id
122121
else:
123122
name = self.asttok.get_text(node.func)
124-
logging.warning("node.func type: %s, name: %s" % (type(node.func), name))
123+
logging.warning(f'node.func type: type(node.func), name: {name}')
125124
name = self.alias2name[name] if name in self.alias2name else name
126125
base = self.name2module[name] if name in self.name2module else None
127126

@@ -139,10 +138,10 @@ def visit_Call(self, node):
139138
args.append('**' + self.asttok.get_text(node.kwargs))
140139

141140
# log stuff
142-
full_name = name if base is None else '%s.%s' % (base, name)
141+
full_name = name if base is None else f'{base}.{name}'
143142

144143
# log stuff
145-
logging.warning("calling function %s with args %s at line %d" % (full_name, args, node.lineno))
144+
logging.warning(f'calling function {full_name} with args {args} at line {node.lineno}')
146145
node_details = {
147146
"Name" : full_name,
148147
"Args" : args,
@@ -200,7 +199,7 @@ def py_astgen(inpath, outfile, configpb, root=None, pkg_name=None, pkg_version=N
200199
try:
201200
all_source = open(infile, 'r').read()
202201
except Exception as e:
203-
logging.warning("Failed to read file %s: %s" % (infile, str(e)))
202+
logging.warning(f'Failed to read {infile}: {str(e)}')
204203
continue
205204

206205
try:
@@ -211,23 +210,23 @@ def py_astgen(inpath, outfile, configpb, root=None, pkg_name=None, pkg_version=N
211210
}
212211
composition["Files"].append(file_details)
213212
except Exception as e:
214-
logging.warning("Failed to parse FILE %s ast details: %s" % (infile, str(e)))
213+
logging.warning(f'Failed to parse {infile} ast details: {str(e)}')
215214

216215
if infile not in infiles:
217216
continue
218217

219218
try:
220219
tree = ast.parse(all_source, filename=infile)
221220
except SyntaxError as se:
222-
logging.warning("Syntax error %s parsing file %s in Python2. Skipping!" % (str(se), infile))
221+
logging.warning(f'Syntax error {str(se)} parsing file {infile} in Python2. Skipping!')
223222
continue
224223

225224
# mark the tree with tokens information
226225
try:
227226
asttok = asttokens.ASTTokens(source_text=all_source, tree=tree, filename=infile)
228227
visitor = PythonDeclRefVisitor(buf=buf, infile=infile, asttok=asttok, configpb=configpb)
229228
visitor.visit(tree)
230-
logging.warning("collected functions: %s" % (Counter(visitor.get_declrefs()).items()))
229+
logging.warning(f'collected functions: {Counter(visitor.get_declrefs()).items()}')
231230

232231
filepb = StaticAnalyzer._get_filepb(infile, root)
233232
for base, name, args, source_text, source_range in visitor.get_declrefs():
@@ -237,11 +236,11 @@ def py_astgen(inpath, outfile, configpb, root=None, pkg_name=None, pkg_version=N
237236
for item_type, item_details in visitor.get_all_declrefs().items():
238237
composition[item_type] += item_details
239238
except Exception as e:
240-
logging.warning("Error parsing AST for file %s in Python3: %s" % (infile, str(se)))
239+
logging.warning(f'Error parsing AST {infile} in Python3: {str(e)}')
241240

242241
# save AST details
243242
try:
244-
logging.warning('writing to %s' % (outfile+'.json'))
243+
logging.warning(f'writing to {outfile}.json')
245244
write_dict_to_file(composition, outfile + '.json')
246245
except Exception as e:
247246
logging.error(str(e))
@@ -272,7 +271,7 @@ def parse_args(argv):
272271
configpath = args.configpath
273272
configpb = AstLookupConfig()
274273
read_proto_from_file(configpb, configpath, binary=False)
275-
logging.debug("loaded lookup config from %s:\n%s", configpath, configpb)
274+
logging.debug(f'loaded lookup config from {configpath}:\n{configpb}')
276275

277276
# Run the ast generation
278277
py_astgen(inpath=args.inpath, outfile=args.outfile, configpb=configpb, root=args.root, pkg_name=args.package_name,

‎packj/audit/static_proxy/astgen_py3.py‎

Lines changed: 27 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -50,28 +50,28 @@ def generic_visit(self, node):
5050
ast.NodeVisitor.generic_visit(self, node)
5151
if self.debug:
5252
if hasattr(node, 'lineno'):
53-
logging.warning('visiting %s node at line %d' % (type(node).__name__, node.lineno))
53+
logging.warning(f'visiting {type(node).__name__} node at line {node.lineno}')
5454
else:
55-
logging.warning('visiting %s node' % (type(node).__name__))
55+
logging.warning(f'visiting {type(node).__name__} node')
5656

5757
def visit_ImportFrom(self, node):
58-
logging.debug('visiting ImportFrom node (line %d)' % (node.lineno))
58+
logging.debug(f'visiting ImportFrom node (line {node.lineno})')
5959
for name in node.names:
6060
self.name2module.setdefault(name.name, node.module)
6161
if name.asname is not None:
6262
self.alias2name.setdefault(name.asname, name.name)
6363
ast.NodeVisitor.generic_visit(self, node)
6464

6565
def visit_FunctionDef(self, node):
66-
logging.debug('visiting FunctionDef node (line %d)' % (node.lineno))
66+
logging.debug(f'visiting FunctionDef node (line {node.lineno})')
6767

6868
# FIXME: warn about redefined functions?
6969
if node.name in self.alias2name or node.name in self.name2module:
70-
logging.warning("redefined imported function %s!" % (node.name))
70+
logging.warning(f'redefined imported function {node.name}!')
7171

7272
ast.NodeVisitor.generic_visit(self, node)
7373
if self.save_feature:
74-
logging.warning("set root_nodes")
74+
logging.warning('set root_nodes')
7575

7676
args = []
7777
for arg in node.args.args:
@@ -85,10 +85,10 @@ def visit_FunctionDef(self, node):
8585
self.all_declrefs["Functions"].append(node_details)
8686

8787
def visit_ClassDef(self, node):
88-
logging.debug('visiting ClassDef node (line %d)' % (node.lineno))
88+
logging.debug(f'visiting ClassDef node (line {node.lineno})')
8989
ast.NodeVisitor.generic_visit(self, node)
9090
if self.save_feature:
91-
logging.warning("set root_nodes")
91+
logging.warning('set root_nodes')
9292

9393
node_details = {
9494
"Name" : node.name,
@@ -98,17 +98,17 @@ def visit_ClassDef(self, node):
9898
self.all_declrefs["Classes"].append(node_details)
9999

100100
def visit_Call(self, node):
101-
logging.debug('visiting Call node (line %d)' % (node.lineno))
101+
logging.debug(f'visiting Call node (line {node.lineno})')
102102

103103
# debug code
104104
if self.debug:
105105
for fieldname, value in ast.iter_fields(node):
106-
logging.warning('fieldname %s, value %s' % (fieldname, value))
106+
logging.warning(f'fieldname {fieldname}, value {value}')
107107
if fieldname == 'func':
108108
for f_fieldname, f_value in ast.iter_fields(value):
109-
logging.info('func fieldname %s, func value %s' % (f_fieldname, f_value))
109+
logging.info(f'func fieldname {f_fieldname}, func value {f_value}')
110110
if f_fieldname == 'id':
111-
logging.warning('func id: %s' % (f_value))
111+
logging.warning(f'func id: {f_value}')
112112

113113
# compute base and func
114114
if isinstance(node.func, ast.Attribute):
@@ -118,24 +118,23 @@ def visit_Call(self, node):
118118
base = node.func.value.id
119119
elif isinstance(node.func.value, ast.Call):
120120
base = self.asttok.get_text(node.func.value)
121-
logging.debug("node.func.value is ast.Call, Ignoring!")
121+
logging.debug('node.func.value is ast.Call, Ignoring!')
122122
elif isinstance(node.func.value, ast.Subscript):
123123
base = self.asttok.get_text(node.func.value)
124124
# NOTE: currently, we use text of chained functions (i.e. foo().bar(), foo() is used),
125125
# because Python is runtime type language, and it is not possible to get the type statically
126-
logging.warning("node.func.value type ast.Subscript, fields: %s",
127-
list(ast.iter_fields(node.func.value)))
126+
logging.warning(f'node.func.value type ast.Subscript, fields: {list(ast.iter_fields(node.func.value))}')
128127
else:
129128
base = self.asttok.get_text(node.func.value)
130-
logging.warning("node.func.value type: %s, fields: %s",
131-
type(node.func.value), list(ast.iter_fields(node.func.value)))
129+
logging.warning(f'node.func.value type: {type(node.func.value)}, \
130+
fields: {list(ast.iter_fields(node.func.value))}')
132131
else:
133132
# NOTE: we assume the imported functions are not redefined! this may not be true!
134133
if isinstance(node.func, ast.Name):
135134
name = node.func.id
136135
else:
137136
name = self.asttok.get_text(node.func)
138-
logging.warning("node.func type: %s, name: %s" % (type(node.func), name))
137+
logging.warning(f'node.func type: {type(node.func)}, name: {name}')
139138
name = self.alias2name[name] if name in self.alias2name else name
140139
base = self.name2module[name] if name in self.name2module else None
141140

@@ -152,10 +151,10 @@ def visit_Call(self, node):
152151
# append '**' to reproduce the calling text
153152
args.append('**' + self.asttok.get_text(node.kwargs))
154153

155-
full_name = name if base is None else '%s.%s' % (base, name)
154+
full_name = name if base is None else f'{base}.{name}'
156155

157156
# log stuff
158-
logging.warning("calling function %s with args %s at line %d" % (full_name, args, node.lineno))
157+
logging.warning(f'calling function {full_name} with args {args} at line {node.lineno}')
159158

160159
node_details = {
161160
"Name" : full_name,
@@ -213,7 +212,7 @@ def py3_astgen(inpath, outfile, configpb, root=None, pkg_name=None, pkg_version=
213212
try:
214213
all_source = open(infile, 'r').read()
215214
except Exception as e:
216-
logging.warning("Failed to read file %s: %s" % (infile, str(e)))
215+
logging.warning(f'Failed to read file {infile}: {str(e)}')
217216
continue
218217

219218
try:
@@ -224,23 +223,23 @@ def py3_astgen(inpath, outfile, configpb, root=None, pkg_name=None, pkg_version=
224223
}
225224
composition["Files"].append(file_details)
226225
except Exception as e:
227-
logging.error("Failed to parse FILE %s ast details: %s" % (infile, str(e)))
226+
logging.error(f'Failed to parse FILE {infile} ast details: {str(e)}')
228227

229228
if infile not in infiles:
230229
continue
231230

232231
try:
233232
tree = ast.parse(all_source, filename=infile)
234233
except SyntaxError as se:
235-
logging.warning("Syntax error %s parsing file %s in Python3. Skipping!" % (str(se), infile))
234+
logging.warning(f'Syntax error {str(se)} parsing file {infile} in Python3. Skipping!')
236235
continue
237236

238237
# mark the tree with tokens information
239238
try:
240239
asttok = asttokens.ASTTokens(source_text=all_source, tree=tree, filename=infile)
241240
visitor = PythonDeclRefVisitor(infile=infile, asttok=asttok, configpb=configpb)
242241
visitor.visit(tree)
243-
logging.warning("collected functions: %s" % (Counter(visitor.get_declrefs()).items()))
242+
logging.warning(f'collected functions: {Counter(visitor.get_declrefs()).items()}')
244243

245244
filepb = StaticAnalyzer._get_filepb(infile, root)
246245
for base, name, args, source_text, source_range in visitor.get_declrefs():
@@ -250,14 +249,14 @@ def py3_astgen(inpath, outfile, configpb, root=None, pkg_name=None, pkg_version=
250249
for item_type, item_details in visitor.get_all_declrefs().items():
251250
composition[item_type] += item_details
252251
except Exception as e:
253-
logging.debug("Error parsing AST for file %s in Python3: %s" % (infile, str(se)))
252+
logging.debug(f'Error parsing AST for file {infile} in Python3: {str(e)}')
254253

255254
# save AST details
256255
try:
257-
logging.warning('writing to %s' % (outfile+'.json'))
256+
logging.warning(f'writing to {outfile}.json')
258257
write_dict_to_file(composition, outfile + '.json')
259258
except Exception as e:
260-
logging.debug("failed to write ast_details: %s" % (str(e)))
259+
logging.debug(f'failed to write ast_details: {str(e)}')
261260

262261
# save resultpb
263262
write_proto_to_file(resultpb, outfile, binary=False)
@@ -283,7 +282,7 @@ def parse_args(argv):
283282
configpath = args.configpath
284283
configpb = AstLookupConfig()
285284
read_proto_from_file(configpb, configpath, binary=False)
286-
logging.debug("loaded lookup config from %s:\n%s" % (configpath, configpb))
285+
logging.debug(f'loaded lookup config from {configpath}:\n{configpb}')
287286

288287
# Run the ast generation
289288
py3_astgen(inpath=args.inpath, outfile=args.outfile, configpb=configpb, root=args.root, pkg_name=args.package_name,

‎packj/audit/static_proxy/py_analyzer.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,9 @@ def exec_py2_astgen(analyze_path, outfile, configpath, root=None, pkg_name=None,
3434
if pkg_version is not None:
3535
astgen_py2_cmd.extend(['-v', pkg_version])
3636
stdout, stderr, error = exec_command("python2 astgen", astgen_py2_cmd, cwd="static_proxy", redirect_mask=3)
37-
assert not error, "could not generate AST"
37+
assert not error, 'could not generate AST'
3838
except Exception as e:
39-
logging.debug("Failed to analyze for APIs using Python2: %s" % (str(e)))
39+
logging.debug(f'Failed to analyze for APIs using Python2: {str(e)}')
4040

4141
def astgen(self, inpath, outfile, root=None, configpath=None, pkg_name=None, pkg_version=None, evaluate_smt=False):
4242
analyze_path, is_decompress_path, outfile, root, configpath = self._sanitize_astgen_args(
@@ -48,17 +48,17 @@ def astgen(self, inpath, outfile, root=None, configpath=None, pkg_name=None, pkg
4848
configpb = AstLookupConfig()
4949
read_proto_from_file(configpb, configpath, binary=False)
5050

51-
logging.debug("loaded lookup config from %s:\n%s", configpath, configpb)
51+
logging.debug(f'loaded lookup config from {configpath}:\n{configpb}')
5252

5353
# invoke the language specific ast generators to call functions
5454
py3_astgen(inpath=analyze_path, outfile=outfile, configpb=configpb, root=root, pkg_name=pkg_name, pkg_version=pkg_version)
5555

5656
# try python2
5757
except SyntaxError as se:
58-
logging.debug("Syntax error %s, now trying to parse %s again in python2!", se, analyze_path)
58+
logging.debug(f'Syntax error {str(se)}, now trying to parse {analyze_path} again in python2!')
5959
self.exec_py2_astgen(analyze_path, outfile, configpath, root=root, pkg_name=pkg_name, pkg_version=pkg_version)
6060
except Exception as e:
61-
logging.debug("Fatal error %s running astgen for %s!" % (str(e), analyze_path))
61+
logging.debug(f'Fatal error {str(e)} running astgen for {analyze_path}!')
6262

6363
# clean up residue files
6464
self._cleanup_astgen(analyze_path=analyze_path, is_decompress_path=is_decompress_path)

0 commit comments

Comments
 (0)