feat(WIP): python parser

.gitignore

@@ -75,3 +75,6 @@ src/lang/testdata
 
 tools
 abcoder
+
+/*.txt
+/*.json

README.md

@@ -52,6 +52,7 @@ ABCoder currently supports the following languages:
 | Go       | ✅           | ✅           |
 | Rust     | ✅           | Coming Soon |
 | C        | Coming Soon | ❌           |
+| Python   | Coming Soon | ❌           |
 
 
 

lang/collect/collect.go

@@ -22,8 +22,10 @@ import (
 	"strings"
 	"unicode"
 
+	"github.com/cloudwego/abcoder/lang/cxx"
 	"github.com/cloudwego/abcoder/lang/log"
 	. "github.com/cloudwego/abcoder/lang/lsp"
+	"github.com/cloudwego/abcoder/lang/python"
 	"github.com/cloudwego/abcoder/lang/rust"
 	"github.com/cloudwego/abcoder/lang/uniast"
 )
@@ -35,8 +37,13 @@ type CollectOption struct {
 	NoNeedComment      bool
 	NeedTest           bool
 	Excludes           []string
+	CacheResults       bool
 }
 
+const (
+	SUPRESS_COLLECT_ERRORS = true
+)
+
 type Collector struct {
 	cli  *LSPClient
 	spec LanguageSpec
@@ -79,6 +86,10 @@ func switchSpec(l uniast.Language) LanguageSpec {
 	switch l {
 	case uniast.Rust:
 		return &rust.RustSpec{}
+	case uniast.Cxx:
+		return &cxx.CxxSpec{}
+	case uniast.Python:
+		return &python.PythonSpec{}
 	default:
 		panic(fmt.Sprintf("unsupported language %s", l))
 	}
@@ -101,6 +112,24 @@ func NewCollector(repo string, cli *LSPClient) *Collector {
 }
 
 func (c *Collector) Collect(ctx context.Context) error {
+	// Example code to configure the LSP client
+	if !c.NeedStdSymbol {
+		if c.Language == uniast.Python {
+			conf := map[string]interface{}{
+				"settings": map[string]interface{}{
+					"pylsp": map[string]interface{}{
+						"plugins": map[string]interface{}{
+							"jedi_definition": map[string]interface{}{
+								"follow_builtin_definitions": false,
+							},
+						},
+					},
+				},
+			}
+			c.cli.Notify(ctx, "workspace/didChangeConfiguration", conf)
+		}
+	}
+
 	excludes := make([]string, len(c.Excludes))
 	for i, e := range c.Excludes {
 		if !filepath.IsAbs(e) {
@@ -111,8 +140,8 @@ func (c *Collector) Collect(ctx context.Context) error {
 	}
 
 	// scan all files
-	roots := make([]*DocumentSymbol, 0, 1024)
-	scanner := func(path string, info os.FileInfo, err error) error {
+	collect_paths := make([]string, 0, 1024)
+	if err := filepath.Walk(c.repo, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -127,20 +156,35 @@ func (c *Collector) Collect(ctx context.Context) error {
 		if c.spec.ShouldSkip(path) {
 			return nil
 		}
+		collect_paths = append(collect_paths, path)
+		return nil
+	}); err != nil {
+		return err
+	}
 
-		// collect symbols
+	// collect symbols
+	roots := make([]*DocumentSymbol, 0, 1024)
+	for i, path := range collect_paths {
 		uri := NewURI(path)
 		symbols, err := c.cli.DocumentSymbols(ctx, uri)
 		if err != nil {
 			return err
 		}
+		log.Info("collecting %d/%d files %s, has %d symbols\n", i, len(collect_paths), path, len(symbols))
 		// file := filepath.Base(path)
+		n_sym := 0
 		for _, sym := range symbols {
+			log.Debug("  collecting symbol %d/%d %s\n", n_sym, len(symbols), sym.Name)
+			n_sym++
 			// collect content
 			content, err := c.cli.Locate(sym.Location)
 			if err != nil {
 				return err
 			}
+			// HACK: skip imported symbols
+			if c.Language == uniast.Python && (strings.HasPrefix(content, "from ") || strings.HasPrefix(content, "import ")) {
+				continue
+			}
 			// collect tokens
 			tokens, err := c.cli.SemanticTokens(ctx, sym.Location)
 			if err != nil {
@@ -151,12 +195,8 @@ func (c *Collector) Collect(ctx context.Context) error {
 			c.syms[sym.Location] = sym
 			roots = append(roots, sym)
 		}
-
-		return nil
-	}
-	if err := filepath.Walk(c.repo, scanner); err != nil {
-		return err
 	}
+	log.Info("collected %d root symbols. going to collect more syms and dependencies...\n", len(roots))
 
 	// collect some extra metadata
 	syms := make([]*DocumentSymbol, 0, len(roots))
@@ -167,6 +207,7 @@ func (c *Collector) Collect(ctx context.Context) error {
 		}
 		c.processSymbol(ctx, sym, 1)
 	}
+	log.Info("collected %d symbols. going to collect dependencies...\n", len(c.syms))
 
 	// collect internal references
 	// for _, sym := range syms {
@@ -200,8 +241,11 @@ func (c *Collector) Collect(ctx context.Context) error {
 	// 	}
 	// }
 
+	num_edges := 0
 	// collect dependencies
-	for _, sym := range syms {
+	for i, sym := range syms {
+		log.Info("collecting dependencies %d/%d %s\n", i, len(syms), sym.Name)
+
 	next_token:
 
 		for i, token := range sym.Tokens {
@@ -247,7 +291,9 @@ func (c *Collector) Collect(ctx context.Context) error {
 			// go to definition
 			dep, err := c.getSymbolByToken(ctx, token)
 			if err != nil || dep == nil {
-				log.Error("dep token %v not found: %v\n", token, err)
+				if !SUPRESS_COLLECT_ERRORS {
+					log.Error("dep token %v not found: %v\n", token, err)
+				}
 				continue
 			}
 
@@ -268,6 +314,8 @@ func (c *Collector) Collect(ctx context.Context) error {
 				c.syms[dep.Location] = dep
 			}
 
+			log.Debug("  Collect: dep %s -> %s (file: %s -> %s)\n", sym.Name, dep.Name, sym.Location, token.Location)
+			num_edges++
 			c.deps[sym] = append(c.deps[sym], dependency{
 				Location: token.Location,
 				Symbol:   dep,
@@ -276,6 +324,7 @@ func (c *Collector) Collect(ctx context.Context) error {
 		}
 	}
 
+	log.Info("collected %d symbols, %d edges.\n", len(c.syms), num_edges)
 	return nil
 }
 
@@ -297,18 +346,31 @@ func (c *Collector) getSymbolByTokenWithLimit(ctx context.Context, tok Token, de
 		return nil, fmt.Errorf("definition of token %s not found", tok)
 	}
 	if len(defs) > 1 {
-		log.Error("definition of token %s not unique", tok)
+		if !SUPRESS_COLLECT_ERRORS {
+			log.Error("definition of token %s not unique", tok)
+		}
 	}
 	return c.getSymbolByLocation(ctx, defs[0], depth, tok)
 }
 
 func (c *Collector) filterEntitySymbols(syms []*DocumentSymbol) *DocumentSymbol {
+	// Choose the most specific symbol
+	var mostSpecific *DocumentSymbol
+	mostSpecific = nil
 	for _, sym := range syms {
-		if c.spec.IsEntitySymbol(*sym) {
-			return sym
+		if !c.spec.IsEntitySymbol(*sym) {
+			continue
+		}
+		if mostSpecific == nil || mostSpecific.Location.Include(sym.Location) {
+			// replace most specific
+			mostSpecific = sym
+		} else if sym.Location.Include(mostSpecific.Location) {
+			// retain most specific
+		} else {
+			log.Error("multiple symbols %s and %s not include each other", mostSpecific, sym)
 		}
 	}
-	return nil
+	return mostSpecific
 }
 
 // return a language entity symbol
@@ -488,7 +550,9 @@ func (c *Collector) getDepsWithLimit(ctx context.Context, sym *DocumentSymbol, t
 	for _, tp := range tps {
 		dep, err := c.getSymbolByTokenWithLimit(ctx, sym.Tokens[tp], depth)
 		if err != nil || sym == nil {
-			log.Error_skip(1, "token %v not found its symbol: %v", tp, err)
+			if !SUPRESS_COLLECT_ERRORS {
+				log.Error_skip(1, "token %v not found its symbol: %v", tp, err)
+			}
 		} else {
 			d := dependency{sym.Tokens[tp].Location, dep}
 			tsyms[tp] = d
@@ -525,7 +589,7 @@ func (c *Collector) collectImpl(ctx context.Context, sym *DocumentSymbol, depth
 		impl = ChunkHead(sym.Text, sym.Location.Range.Start, sym.Tokens[fn].Location.Range.Start)
 	}
 	if impl == "" || len(impl) < len(sym.Name) {
-		impl = sym.Name
+		impl = fmt.Sprintf("class %s {\n", sym.Name)
 	}
 	// search all methods
 	for _, method := range c.syms {
@@ -581,12 +645,16 @@ func (c *Collector) processSymbol(ctx context.Context, sym *DocumentSymbol, dept
 			}
 		}
 		if i < 0 || i >= len(sym.Tokens) {
-			log.Error("get type token of variable symbol %s failed\n", sym)
+			if !SUPRESS_COLLECT_ERRORS {
+				log.Error("get type token of variable symbol %s failed\n", sym)
+			}
 			return
 		}
 		tsym, err := c.getSymbolByTokenWithLimit(ctx, sym.Tokens[i], depth-1)
 		if err != nil || tsym == nil {
-			log.Error("get type symbol for token %s failed:%v\n", sym.Tokens[i], err)
+			if !SUPRESS_COLLECT_ERRORS {
+				log.Error("get type symbol for token %s failed:%v\n", sym.Tokens[i], err)
+			}
 			return
 		}
 		c.vars[sym] = dependency{
@@ -613,9 +681,12 @@ func (c *Collector) updateFunctionInfo(sym *DocumentSymbol, tsyms, ipsyms, opsym
 		}
 	} else {
 		f = functionInfo{
-			TypeParams: tsyms,
-			Inputs:     ipsyms,
-			Outputs:    opsyms,
+			TypeParams:       tsyms,
+			Inputs:           ipsyms,
+			Outputs:          opsyms,
+			InputsSorted:     is,
+			OutputsSorted:    os,
+			TypeParamsSorted: ts,
 		}
 		if rsym != nil {
 			if f.Method == nil {