Create tokens with the right data

internal/ast/ast.go

@@ -10781,6 +10781,8 @@ type SourceFile struct {
 
 	tokenCacheMu     sync.Mutex
 	tokenCache       map[core.TextRange]*Node
+	tokenFactoryMu   sync.Mutex
+	tokenFactory     *NodeFactory
 	declarationMapMu sync.Mutex
 	declarationMap   map[string][]*Node
 }
@@ -10942,6 +10944,16 @@ func (node *SourceFile) GetOrCreateToken(
 	pos int,
 	end int,
 	parent *Node,
+) *TokenNode {
+	return node.GetOrCreateTokenEx(kind, pos, end, parent, TokenFlagsNone)
+}
+
+func (node *SourceFile) GetOrCreateTokenEx(
+	kind Kind,
+	pos int,
+	end int,
+	parent *Node,
+	flags TokenFlags,
 ) *TokenNode {
 	node.tokenCacheMu.Lock()
 	defer node.tokenCacheMu.Unlock()
@@ -10959,13 +10971,49 @@ func (node *SourceFile) GetOrCreateToken(
 		return token
 	}
 
-	token := newNode(kind, &Token{}, NodeFactoryHooks{})
+	token := createToken(kind, node, pos, end, flags)
 	token.Loc = loc
 	token.Parent = parent
 	node.tokenCache[loc] = token
 	return token
 }
 
+// `kind` should be a token kind.
+func createToken(kind Kind, file *SourceFile, pos, end int, flags TokenFlags) *Node {
+	file.tokenFactoryMu.Lock()
+	defer file.tokenFactoryMu.Unlock()
+	if file.tokenFactory == nil {
+		file.tokenFactory = NewNodeFactory(NodeFactoryHooks{})
+	}
+	text := file.text[pos:end]
+	switch kind {
+	case KindNumericLiteral:
+		return file.tokenFactory.NewNumericLiteral(text)
+	case KindBigIntLiteral:
+		return file.tokenFactory.NewBigIntLiteral(text)
+	case KindStringLiteral:
+		return file.tokenFactory.NewStringLiteral(text)
+	case KindJsxText, KindJsxTextAllWhiteSpaces:
+		return file.tokenFactory.NewJsxText(text, kind == KindJsxTextAllWhiteSpaces)
+	case KindRegularExpressionLiteral:
+		return file.tokenFactory.NewRegularExpressionLiteral(text)
+	case KindNoSubstitutionTemplateLiteral:
+		return file.tokenFactory.NewNoSubstitutionTemplateLiteral(text)
+	case KindTemplateHead:
+		return file.tokenFactory.NewTemplateHead(text, "" /*rawText*/, flags)
+	case KindTemplateMiddle:
+		return file.tokenFactory.NewTemplateMiddle(text, "" /*rawText*/, flags)
+	case KindTemplateTail:
+		return file.tokenFactory.NewTemplateTail(text, "" /*rawText*/, flags)
+	case KindIdentifier:
+		return file.tokenFactory.NewIdentifier(text)
+	case KindPrivateIdentifier:
+		return file.tokenFactory.NewPrivateIdentifier(text)
+	default: // Punctuation and keywords
+		return file.tokenFactory.NewToken(kind)
+	}
+}
+
 func IsSourceFile(node *Node) bool {
 	return node.Kind == KindSourceFile
 }

internal/astnav/tokens.go

@@ -187,17 +187,18 @@ func getTokenAtPosition(
 				tokenFullStart := scanner.TokenFullStart()
 				tokenStart := core.IfElse(allowPositionInLeadingTrivia, tokenFullStart, scanner.TokenStart())
 				tokenEnd := scanner.TokenEnd()
+				flags := scanner.TokenFlags()
 				if tokenStart <= position && (position < tokenEnd) {
 					if token == ast.KindIdentifier || !ast.IsTokenKind(token) {
 						if ast.IsJSDocKind(current.Kind) {
 							return current
 						}
 						panic(fmt.Sprintf("did not expect %s to have %s in its trivia", current.Kind.String(), token.String()))
 					}
-					return sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, current)
+					return sourceFile.GetOrCreateTokenEx(token, tokenFullStart, tokenEnd, current, flags)
 				}
 				if includePrecedingTokenAtEndPosition != nil && tokenEnd == position {
-					prevToken := sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, current)
+					prevToken := sourceFile.GetOrCreateTokenEx(token, tokenFullStart, tokenEnd, current, flags)
 					if includePrecedingTokenAtEndPosition(prevToken) {
 						return prevToken
 					}
@@ -514,7 +515,8 @@ func findRightmostValidToken(endPos int, sourceFile *ast.SourceFile, containingN
 					tokenFullStart := scanner.TokenFullStart()
 					tokenEnd := scanner.TokenEnd()
 					startPos = tokenEnd
-					tokens = append(tokens, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, n))
+					flags := scanner.TokenFlags()
+					tokens = append(tokens, sourceFile.GetOrCreateTokenEx(token, tokenFullStart, tokenEnd, n, flags))
 					scanner.Scan()
 				}
 				startPos = visitedNode.End()
@@ -531,7 +533,8 @@ func findRightmostValidToken(endPos int, sourceFile *ast.SourceFile, containingN
 				tokenFullStart := scanner.TokenFullStart()
 				tokenEnd := scanner.TokenEnd()
 				startPos = tokenEnd
-				tokens = append(tokens, sourceFile.GetOrCreateToken(token, tokenFullStart, tokenEnd, n))
+				flags := scanner.TokenFlags()
+				tokens = append(tokens, sourceFile.GetOrCreateTokenEx(token, tokenFullStart, tokenEnd, n, flags))
 				scanner.Scan()
 			}
 
@@ -616,8 +619,9 @@ func FindNextToken(previousToken *ast.Node, parent *ast.Node, file *ast.SourceFi
 			tokenFullStart := scanner.TokenFullStart()
 			tokenStart := scanner.TokenStart()
 			tokenEnd := scanner.TokenEnd()
+			flags := scanner.TokenFlags()
 			if tokenStart == previousToken.End() {
-				return file.GetOrCreateToken(token, tokenFullStart, tokenEnd, n)
+				return file.GetOrCreateTokenEx(token, tokenFullStart, tokenEnd, n, flags)
 			}
 			panic(fmt.Sprintf("Expected to find next token at %d, got token %s at %d", previousToken.End(), token, tokenStart))
 		}
@@ -690,7 +694,8 @@ func FindChildOfKind(containingNode *ast.Node, kind ast.Kind, sourceFile *ast.So
 			tokenKind := scan.Token()
 			tokenFullStart := scan.TokenFullStart()
 			tokenEnd := scan.TokenEnd()
-			token := sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, containingNode)
+			flags := scan.TokenFlags()
+			token := sourceFile.GetOrCreateTokenEx(tokenKind, tokenFullStart, tokenEnd, containingNode, flags)
 			if tokenKind == kind {
 				foundChild = token
 				return true
@@ -720,7 +725,8 @@ func FindChildOfKind(containingNode *ast.Node, kind ast.Kind, sourceFile *ast.So
 		tokenKind := scan.Token()
 		tokenFullStart := scan.TokenFullStart()
 		tokenEnd := scan.TokenEnd()
-		token := sourceFile.GetOrCreateToken(tokenKind, tokenFullStart, tokenEnd, containingNode)
+		flags := scan.TokenFlags()
+		token := sourceFile.GetOrCreateTokenEx(tokenKind, tokenFullStart, tokenEnd, containingNode, flags)
 		if tokenKind == kind {
 			return token
 		}

internal/fourslash/tests/completionsUnterminatedLiteral_test.go

@@ -0,0 +1,25 @@
+package fourslash_test
+
+import (
+	"testing"
+
+	"github.com/microsoft/typescript-go/internal/fourslash"
+	. "github.com/microsoft/typescript-go/internal/fourslash/tests/util"
+	"github.com/microsoft/typescript-go/internal/testutil"
+)
+
+func TestCompletionsUnterminatedLiteral(t *testing.T) {
+	t.Parallel()
+
+	defer testutil.RecoverAndFail(t, "Panic on fourslash test")
+	const content = `// @noLib: true
+function foo(a"/*1*/`
+	f, done := fourslash.NewFourslash(t, nil /*capabilities*/, content)
+	defer done()
+	f.VerifyCompletions(t, "1", &fourslash.CompletionsExpectedList{
+		ItemDefaults: &fourslash.CompletionsExpectedItemDefaults{
+			CommitCharacters: &DefaultCommitCharacters,
+		},
+		Items: &fourslash.CompletionsExpectedItems{},
+	})
+}