diff --git a/internal/ast/ast.go b/internal/ast/ast.go index d1d6d62a0a..bb19b1fce6 100644 --- a/internal/ast/ast.go +++ b/internal/ast/ast.go @@ -10781,6 +10781,7 @@ type SourceFile struct { tokenCacheMu sync.Mutex tokenCache map[core.TextRange]*Node + tokenFactory *NodeFactory declarationMapMu sync.Mutex declarationMap map[string][]*Node } @@ -10942,6 +10943,7 @@ func (node *SourceFile) GetOrCreateToken( pos int, end int, parent *Node, + flags TokenFlags, ) *TokenNode { node.tokenCacheMu.Lock() defer node.tokenCacheMu.Unlock() @@ -10959,13 +10961,57 @@ func (node *SourceFile) GetOrCreateToken( return token } - token := newNode(kind, &Token{}, NodeFactoryHooks{}) + token := createToken(kind, node, pos, end, flags) token.Loc = loc token.Parent = parent node.tokenCache[loc] = token return token } +// `kind` should be a token kind. +func createToken(kind Kind, file *SourceFile, pos, end int, flags TokenFlags) *Node { + if file.tokenFactory == nil { + file.tokenFactory = NewNodeFactory(NodeFactoryHooks{}) + } + text := file.text[pos:end] + switch kind { + case KindNumericLiteral: + literal := file.tokenFactory.NewNumericLiteral(text) + literal.AsNumericLiteral().TokenFlags = flags & TokenFlagsNumericLiteralFlags + return literal + case KindBigIntLiteral: + literal := file.tokenFactory.NewBigIntLiteral(text) + literal.AsBigIntLiteral().TokenFlags = flags & TokenFlagsNumericLiteralFlags + return literal + case KindStringLiteral: + literal := file.tokenFactory.NewStringLiteral(text) + literal.AsStringLiteral().TokenFlags = flags & TokenFlagsStringLiteralFlags + return literal + case KindJsxText, KindJsxTextAllWhiteSpaces: + return file.tokenFactory.NewJsxText(text, kind == KindJsxTextAllWhiteSpaces) + case KindRegularExpressionLiteral: + literal := file.tokenFactory.NewRegularExpressionLiteral(text) + literal.AsRegularExpressionLiteral().TokenFlags = flags & TokenFlagsRegularExpressionLiteralFlags + return literal + case KindNoSubstitutionTemplateLiteral: + literal := file.tokenFactory.NewNoSubstitutionTemplateLiteral(text) + literal.AsNoSubstitutionTemplateLiteral().TokenFlags = flags & TokenFlagsTemplateLiteralLikeFlags + return literal + case KindTemplateHead: + return file.tokenFactory.NewTemplateHead(text, "" /*rawText*/, flags&TokenFlagsTemplateLiteralLikeFlags) + case KindTemplateMiddle: + return file.tokenFactory.NewTemplateMiddle(text, "" /*rawText*/, flags&TokenFlagsTemplateLiteralLikeFlags) + case KindTemplateTail: + return file.tokenFactory.NewTemplateTail(text, "" /*rawText*/, flags&TokenFlagsTemplateLiteralLikeFlags) + case KindIdentifier: + return file.tokenFactory.NewIdentifier(text) + case KindPrivateIdentifier: + return file.tokenFactory.NewPrivateIdentifier(text) + default: // Punctuation and keywords + return file.tokenFactory.NewToken(kind) + } +} + func IsSourceFile(node *Node) bool { return node.Kind == KindSourceFile } diff --git a/internal/astnav/tokens.go b/internal/astnav/tokens.go index 85fa79801e..c4fb9ee350 100644 --- a/internal/astnav/tokens.go +++ b/internal/astnav/tokens.go @@ -187,6 +187,7 @@ func getTokenAtPosition( tokenFullStart := scanner.TokenFullStart() tokenStart := core.IfElse(allowPositionInLeadingTrivia, tokenFullStart, scanner.TokenStart()) tokenEnd := scanner.TokenEnd() + flags := scanner.TokenFlags() if tokenStart <= position && (position < tokenEnd) { if token == ast.KindIdentifier || !ast.IsTokenKind(token) { if ast.IsJSDocKind(current.Kind) { @@ -194,10 +195,10 @@ func getTokenAtPosition( } panic(fmt.Sprintf("did not expect %s to have %s in its trivia", current.Kind.String(), token.String())) } - return sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, current) + return sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, current, flags) } if includePrecedingTokenAtEndPosition != nil && tokenEnd == position { - prevToken := sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, current) + prevToken := sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, current, flags) if includePrecedingTokenAtEndPosition(prevToken) { return prevToken } @@ -514,7 +515,8 @@ func findRightmostValidToken(endPos int, sourceFile *ast.SourceFile, containingN tokenFullStart := scanner.TokenFullStart() tokenEnd := scanner.TokenEnd() startPos = tokenEnd - tokens = append(tokens, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, n)) + flags := scanner.TokenFlags() + tokens = append(tokens, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, n, flags)) scanner.Scan() } startPos = visitedNode.End() @@ -531,7 +533,8 @@ func findRightmostValidToken(endPos int, sourceFile *ast.SourceFile, containingN tokenFullStart := scanner.TokenFullStart() tokenEnd := scanner.TokenEnd() startPos = tokenEnd - tokens = append(tokens, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, n)) + flags := scanner.TokenFlags() + tokens = append(tokens, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, n, flags)) scanner.Scan() } @@ -616,8 +619,9 @@ func FindNextToken(previousToken *ast.Node, parent *ast.Node, file *ast.SourceFi tokenFullStart := scanner.TokenFullStart() tokenStart := scanner.TokenStart() tokenEnd := scanner.TokenEnd() + flags := scanner.TokenFlags() if tokenStart == previousToken.End() { - return file.GetOrCreateToken(token, tokenFullStart, tokenEnd, n) + return file.GetOrCreateToken(token, tokenFullStart, tokenEnd, n, flags) } panic(fmt.Sprintf("Expected to find next token at %d, got token %s at %d", previousToken.End(), token, tokenStart)) } @@ -690,7 +694,8 @@ func FindChildOfKind(containingNode *ast.Node, kind ast.Kind, sourceFile *ast.So tokenKind := scan.Token() tokenFullStart := scan.TokenFullStart() tokenEnd := scan.TokenEnd() - token := sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, containingNode) + flags := scan.TokenFlags() + token := sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, containingNode, flags) if tokenKind == kind { foundChild = token return true @@ -720,7 +725,8 @@ func FindChildOfKind(containingNode *ast.Node, kind ast.Kind, sourceFile *ast.So tokenKind := scan.Token() tokenFullStart := scan.TokenFullStart() tokenEnd := scan.TokenEnd() - token := sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, containingNode) + flags := scan.TokenFlags() + token := sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, containingNode, flags) if tokenKind == kind { return token } diff --git a/internal/fourslash/tests/completionsUnterminatedLiteral_test.go b/internal/fourslash/tests/completionsUnterminatedLiteral_test.go new file mode 100644 index 0000000000..cfc0e32afe --- /dev/null +++ b/internal/fourslash/tests/completionsUnterminatedLiteral_test.go @@ -0,0 +1,25 @@ +package fourslash_test + +import ( + "testing" + + "github.com/microsoft/typescript-go/internal/fourslash" + . "github.com/microsoft/typescript-go/internal/fourslash/tests/util" + "github.com/microsoft/typescript-go/internal/testutil" +) + +func TestCompletionsUnterminatedLiteral(t *testing.T) { + t.Parallel() + + defer testutil.RecoverAndFail(t, "Panic on fourslash test") + const content = `// @noLib: true +function foo(a"/*1*/` + f, done := fourslash.NewFourslash(t, nil /*capabilities*/, content) + defer done() + f.VerifyCompletions(t, "1", &fourslash.CompletionsExpectedList{ + ItemDefaults: &fourslash.CompletionsExpectedItemDefaults{ + CommitCharacters: &DefaultCommitCharacters, + }, + Items: &fourslash.CompletionsExpectedItems{}, + }) +} diff --git a/internal/ls/change/tracker.go b/internal/ls/change/tracker.go index 28c5c65205..cbb80f046d 100644 --- a/internal/ls/change/tracker.go +++ b/internal/ls/change/tracker.go @@ -178,7 +178,7 @@ func (t *Tracker) InsertNodeBefore(sourceFile *ast.SourceFile, before *ast.Node, // InsertModifierBefore inserts a modifier token (like 'type') before a node with a trailing space. func (t *Tracker) InsertModifierBefore(sourceFile *ast.SourceFile, modifier ast.Kind, before *ast.Node) { pos := astnav.GetStartOfNode(before, sourceFile, false) - token := sourceFile.GetOrCreateToken(modifier, pos, pos, before.Parent) + token := sourceFile.GetOrCreateToken(modifier, pos, pos, before.Parent, ast.TokenFlagsNone) t.InsertNodeAt(sourceFile, core.TextPos(pos), token, NodeOptions{Suffix: " "}) } @@ -262,7 +262,7 @@ func (t *Tracker) endPosForInsertNodeAfter(sourceFile *ast.SourceFile, after *as endPos := t.converters.PositionToLineAndCharacter(sourceFile, core.TextPos(after.End())) t.ReplaceRange(sourceFile, lsproto.Range{Start: endPos, End: endPos}, - sourceFile.GetOrCreateToken(ast.KindSemicolonToken, after.End(), after.End(), after.Parent), + sourceFile.GetOrCreateToken(ast.KindSemicolonToken, after.End(), after.End(), after.Parent, ast.TokenFlagsNone), NodeOptions{}, ) } @@ -347,7 +347,7 @@ func (t *Tracker) InsertNodeInListAfter(sourceFile *ast.SourceFile, after *ast.N // insert separator immediately following the 'after' node to preserve comments in trailing trivia // !!! formatcontext - t.ReplaceRange(sourceFile, lsproto.Range{Start: end, End: end}, sourceFile.GetOrCreateToken(separator, after.End(), after.End()+len(separatorString), after.Parent), NodeOptions{}) + t.ReplaceRange(sourceFile, lsproto.Range{Start: end, End: end}, sourceFile.GetOrCreateToken(separator, after.End(), after.End()+len(separatorString), after.Parent, ast.TokenFlagsNone), NodeOptions{}) // use the same indentation as 'after' item indentation := format.FindFirstNonWhitespaceColumn(afterStartLinePosition, afterStart, sourceFile, t.formatSettings) // insert element before the line break on the line that contains 'after' element diff --git a/internal/ls/lsutil/children.go b/internal/ls/lsutil/children.go index 7e6c0d1ed0..b3ea1eb355 100644 --- a/internal/ls/lsutil/children.go +++ b/internal/ls/lsutil/children.go @@ -25,7 +25,7 @@ func GetLastChild(node *ast.Node, sourceFile *ast.SourceFile) *ast.Node { tokenKind := scanner.Token() tokenFullStart := scanner.TokenFullStart() tokenEnd := scanner.TokenEnd() - lastToken = sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, node) + lastToken = sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, node, scanner.TokenFlags()) startPos = tokenEnd scanner.Scan() } @@ -108,7 +108,7 @@ func GetFirstToken(node *ast.Node, sourceFile *ast.SourceFile) *ast.Node { tokenKind := scanner.Token() tokenFullStart := scanner.TokenFullStart() tokenEnd := scanner.TokenEnd() - firstToken = sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, node) + firstToken = sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, node, scanner.TokenFlags()) } if firstToken != nil { diff --git a/internal/ls/signaturehelp.go b/internal/ls/signaturehelp.go index 7953066676..2335b27a0e 100644 --- a/internal/ls/signaturehelp.go +++ b/internal/ls/signaturehelp.go @@ -1241,7 +1241,7 @@ func getTokenFromNodeList(nodeList *ast.NodeList, nodeListParent *ast.Node, sour token := scanner.Token() tokenFullStart := scanner.TokenFullStart() tokenEnd := scanner.TokenEnd() - tokens = append(tokens, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, nodeListParent)) + tokens = append(tokens, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, nodeListParent, scanner.TokenFlags())) left = tokenEnd } } diff --git a/internal/ls/utilities.go b/internal/ls/utilities.go index 2778243905..0760b90e1d 100644 --- a/internal/ls/utilities.go +++ b/internal/ls/utilities.go @@ -752,7 +752,7 @@ func nodeEndsWith(n *ast.Node, expectedLastToken ast.Kind, sourceFile *ast.Sourc tokenKind := scanner.Token() tokenFullStart := scanner.TokenFullStart() tokenEnd := scanner.TokenEnd() - token := sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, n) + token := sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, n, scanner.TokenFlags()) lastNodeAndTokens = append(lastNodeAndTokens, token) startPos = tokenEnd scanner.Scan() @@ -1571,7 +1571,7 @@ func getChildrenFromNonJSDocNode(node *ast.Node, sourceFile *ast.SourceFile) []* token := scanner.Token() tokenFullStart := scanner.TokenFullStart() tokenEnd := scanner.TokenEnd() - children = append(children, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, node)) + children = append(children, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, node, scanner.TokenFlags())) pos = tokenEnd scanner.Scan() } @@ -1583,7 +1583,7 @@ func getChildrenFromNonJSDocNode(node *ast.Node, sourceFile *ast.SourceFile) []* token := scanner.Token() tokenFullStart := scanner.TokenFullStart() tokenEnd := scanner.TokenEnd() - children = append(children, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, node)) + children = append(children, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, node, scanner.TokenFlags())) pos = tokenEnd scanner.Scan() }