diff --git a/runtime/ast/block.go b/runtime/ast/block.go index 1f1b70a596..69518f8077 100644 --- a/runtime/ast/block.go +++ b/runtime/ast/block.go @@ -29,10 +29,18 @@ import ( type Block struct { Statements []Statement Range + Comments } var _ Element = &Block{} +// TODO(preserve-comments): Migrate and remove +func NewBlockWithComments(memoryGauge common.MemoryGauge, statements []Statement, astRange Range, comments Comments) *Block { + block := NewBlock(memoryGauge, statements, astRange) + block.Comments = comments + return block +} + func NewBlock(memoryGauge common.MemoryGauge, statements []Statement, astRange Range) *Block { common.UseMemory(memoryGauge, common.BlockMemoryUsage) diff --git a/runtime/ast/comments.go b/runtime/ast/comments.go new file mode 100644 index 0000000000..4081e074df --- /dev/null +++ b/runtime/ast/comments.go @@ -0,0 +1,71 @@ +package ast + +import ( + "bytes" + "github.com/onflow/cadence/runtime/common" +) + +type Comments struct { + Leading []Comment `json:"-"` + Trailing []Comment `json:"-"` +} + +type Comment struct { + source []byte +} + +func NewComment(memoryGauge common.MemoryGauge, source []byte) Comment { + // TODO(preserve-comments): Track memory usage + return Comment{ + source: source, + } +} + +var blockCommentDocStringPrefix = []byte("/**") +var blockCommentStringPrefix = []byte("/*") +var lineCommentDocStringPrefix = []byte("///") +var lineCommentStringPrefix = []byte("//") +var blockCommentStringSuffix = []byte("*/") + +func (c Comment) Multiline() bool { + return bytes.HasPrefix(c.source, blockCommentStringPrefix) +} + +func (c Comment) Doc() bool { + if c.Multiline() { + return bytes.HasPrefix(c.source, blockCommentDocStringPrefix) + } else { + return bytes.HasPrefix(c.source, lineCommentDocStringPrefix) + } +} + +// Text without opening/closing comment characters /*, /**, */, // +func (c Comment) Text() []byte { + withoutPrefixes := cutOptionalPrefixes(c.source, [][]byte{ + blockCommentDocStringPrefix, // must be before blockCommentStringPrefix + blockCommentStringPrefix, + lineCommentDocStringPrefix, // must be before lineCommentStringPrefix + lineCommentStringPrefix, + }) + return cutOptionalSuffixes(withoutPrefixes, [][]byte{ + blockCommentStringSuffix, + }) +} + +func cutOptionalPrefixes(input []byte, prefixes [][]byte) (output []byte) { + output = input + for _, prefix := range prefixes { + cut, _ := bytes.CutPrefix(output, prefix) + output = cut + } + return +} + +func cutOptionalSuffixes(input []byte, suffixes [][]byte) (output []byte) { + output = input + for _, suffix := range suffixes { + cut, _ := bytes.CutSuffix(output, suffix) + output = cut + } + return +} diff --git a/runtime/ast/function_declaration.go b/runtime/ast/function_declaration.go index f86f6be670..2614edba08 100644 --- a/runtime/ast/function_declaration.go +++ b/runtime/ast/function_declaration.go @@ -20,6 +20,7 @@ package ast import ( "encoding/json" + "strings" "github.com/turbolent/prettier" @@ -69,17 +70,51 @@ type FunctionDeclaration struct { ParameterList *ParameterList ReturnTypeAnnotation *TypeAnnotation FunctionBlock *FunctionBlock - DocString string Identifier Identifier StartPos Position `json:"-"` Access Access Flags FunctionDeclarationFlags + Comments } var _ Element = &FunctionDeclaration{} var _ Declaration = &FunctionDeclaration{} var _ Statement = &FunctionDeclaration{} +// TODO(preserve-comments): Temporary, add `comments` param to NewFunctionDeclaration in the future +func NewFunctionDeclarationWithComments( + gauge common.MemoryGauge, + access Access, + purity FunctionPurity, + isStatic bool, + isNative bool, + identifier Identifier, + typeParameterList *TypeParameterList, + parameterList *ParameterList, + returnTypeAnnotation *TypeAnnotation, + functionBlock *FunctionBlock, + startPos Position, + docString string, + comments Comments, +) *FunctionDeclaration { + decl := NewFunctionDeclaration( + gauge, + access, + purity, + isStatic, + isNative, + identifier, + typeParameterList, + parameterList, + returnTypeAnnotation, + functionBlock, + startPos, + docString, + ) + decl.Comments = comments + return decl +} + func NewFunctionDeclaration( gauge common.MemoryGauge, access Access, @@ -114,7 +149,6 @@ func NewFunctionDeclaration( ReturnTypeAnnotation: returnTypeAnnotation, FunctionBlock: functionBlock, StartPos: startPos, - DocString: docString, } } @@ -176,7 +210,16 @@ func (d *FunctionDeclaration) DeclarationMembers() *Members { } func (d *FunctionDeclaration) DeclarationDocString() string { - return d.DocString + var s strings.Builder + for _, comment := range d.Comments.Leading { + if comment.Doc() { + if s.Len() > 0 { + s.WriteRune('\n') + } + s.Write(comment.Text()) + } + } + return s.String() } func (d *FunctionDeclaration) Doc() prettier.Doc { @@ -200,16 +243,18 @@ func (d *FunctionDeclaration) MarshalJSON() ([]byte, error) { *Alias Type string Range - IsStatic bool - IsNative bool - Flags FunctionDeclarationFlags `json:",omitempty"` + IsStatic bool + IsNative bool + Flags FunctionDeclarationFlags `json:",omitempty"` + DocString string }{ - Type: "FunctionDeclaration", - Range: NewUnmeteredRangeFromPositioned(d), - IsStatic: d.IsStatic(), - IsNative: d.IsNative(), - Alias: (*Alias)(d), - Flags: 0, + Type: "FunctionDeclaration", + Range: NewUnmeteredRangeFromPositioned(d), + IsStatic: d.IsStatic(), + IsNative: d.IsNative(), + Alias: (*Alias)(d), + Flags: 0, + DocString: d.DeclarationDocString(), }) } diff --git a/runtime/ast/function_declaration_test.go b/runtime/ast/function_declaration_test.go index c164a67da4..521df5bf8b 100644 --- a/runtime/ast/function_declaration_test.go +++ b/runtime/ast/function_declaration_test.go @@ -113,8 +113,10 @@ func TestFunctionDeclaration_MarshalJSON(t *testing.T) { }, }, }, - DocString: "test", - StartPos: Position{Offset: 34, Line: 35, Column: 36}, + Comments: Comments{ + Leading: []Comment{NewComment(nil, []byte("///test"))}, + }, + StartPos: Position{Offset: 34, Line: 35, Column: 36}, } actual, err := json.Marshal(decl) @@ -584,8 +586,10 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { }, }, }, - DocString: "test", - StartPos: Position{Offset: 34, Line: 35, Column: 36}, + Comments: Comments{ + Leading: []Comment{NewComment(nil, []byte("///test"))}, + }, + StartPos: Position{Offset: 34, Line: 35, Column: 36}, }, } diff --git a/runtime/ast/identifier.go b/runtime/ast/identifier.go index 515ea7699b..dff0b10739 100644 --- a/runtime/ast/identifier.go +++ b/runtime/ast/identifier.go @@ -20,7 +20,6 @@ package ast import ( "encoding/json" - "github.com/onflow/cadence/runtime/common" ) diff --git a/runtime/ast/position.go b/runtime/ast/position.go index 3a7dc42f89..1183985a9f 100644 --- a/runtime/ast/position.go +++ b/runtime/ast/position.go @@ -143,6 +143,12 @@ func (e Range) EndPosition(common.MemoryGauge) Position { return e.EndPos } +func (e Range) Source(input []byte) []byte { + startOffset := e.StartPos.Offset + endOffset := e.EndPos.Offset + 1 + return input[startOffset:endOffset] +} + // NewRangeFromPositioned func NewRangeFromPositioned(memoryGauge common.MemoryGauge, hasPosition HasPosition) Range { diff --git a/runtime/old_parser/declaration_test.go b/runtime/old_parser/declaration_test.go index 1ecbc1e9bb..a009603a0d 100644 --- a/runtime/old_parser/declaration_test.go +++ b/runtime/old_parser/declaration_test.go @@ -816,8 +816,10 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " Test", - StartPos: ast.Position{Line: 2, Column: 0, Offset: 9}, + Comments: ast.Comments{ + Leading: []ast.Comment{ast.NewComment(nil, []byte("/// Test"))}, + }, + StartPos: ast.Position{Line: 2, Column: 0, Offset: 9}, }, }, result, @@ -854,8 +856,13 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " First line\n Second line", - StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// First line")), + ast.NewComment(nil, []byte("/// Second line")), + }, + }, + StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, }, }, result, @@ -892,8 +899,12 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " Cool dogs.\n\n Cool cats!! ", - StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/** Cool dogs.\n\n Cool cats!! */")), + }, + }, + StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, }, }, result, @@ -6870,8 +6881,12 @@ func TestParseMemberDocStrings(t *testing.T) { Members: ast.NewUnmeteredMembers( []ast.Declaration{ &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " noReturnNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// noReturnNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "noReturnNoBlock", Pos: ast.Position{Offset: 78, Line: 5, Column: 18}, @@ -6885,8 +6900,12 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 74, Line: 5, Column: 14}, }, &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " returnNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// returnNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "returnNoBlock", Pos: ast.Position{Offset: 147, Line: 8, Column: 18}, @@ -6910,8 +6929,12 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 143, Line: 8, Column: 14}, }, &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " returnAndBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// returnAndBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "returnAndBlock", Pos: ast.Position{Offset: 220, Line: 11, Column: 18}, @@ -6938,6 +6961,10 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 245, Line: 11, Column: 43}, EndPos: ast.Position{Offset: 246, Line: 11, Column: 44}, }, + Comments: ast.Comments{ + Leading: []ast.Comment{}, + Trailing: []ast.Comment{}, + }, }, }, StartPos: ast.Position{Offset: 216, Line: 11, Column: 14}, @@ -6988,8 +7015,12 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindUnknown, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " unknown", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// unknown")), + }, + }, Identifier: ast.Identifier{ Identifier: "unknown", Pos: ast.Position{Offset: 66, Line: 5, Column: 14}, @@ -7006,8 +7037,12 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindInitializer, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " initNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// initNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "init", Pos: ast.Position{Offset: 121, Line: 8, Column: 14}, @@ -7024,8 +7059,12 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindDestructorLegacy, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " destroyWithBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// destroyWithBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "destroy", Pos: ast.Position{Offset: 178, Line: 11, Column: 14}, diff --git a/runtime/old_parser/function.go b/runtime/old_parser/function.go index 7d0054faaf..9e02144775 100644 --- a/runtime/old_parser/function.go +++ b/runtime/old_parser/function.go @@ -295,8 +295,8 @@ func parseFunctionDeclaration( nativePos *ast.Position, docString string, ) (*ast.FunctionDeclaration, error) { - - startPos := ast.EarliestPosition(p.current.StartPos, accessPos, staticPos, nativePos) + startToken := p.current + startPos := ast.EarliestPosition(startToken.StartPos, accessPos, staticPos, nativePos) // Skip the `fun` keyword p.nextSemanticToken() @@ -329,7 +329,7 @@ func parseFunctionDeclaration( return nil, err } - return ast.NewFunctionDeclaration( + return ast.NewFunctionDeclarationWithComments( p.memoryGauge, access, ast.FunctionPurityUnspecified, @@ -342,6 +342,7 @@ func parseFunctionDeclaration( functionBlock, startPos, docString, + p.newCommentsFromTrivia(startToken.LeadingTrivia, []lexer.Trivia{}), ), nil } diff --git a/runtime/old_parser/parser.go b/runtime/old_parser/parser.go index 05b0d5bf84..299109886a 100644 --- a/runtime/old_parser/parser.go +++ b/runtime/old_parser/parser.go @@ -274,6 +274,11 @@ func (p *parser) tokenSource(token lexer.Token) []byte { return token.Source(input) } +func (p *parser) triviaSource(trivia lexer.Trivia) []byte { + input := p.tokens.Input() + return trivia.Source(input) +} + func (p *parser) currentTokenSource() []byte { return p.tokenSource(p.current) } @@ -527,6 +532,32 @@ func (p *parser) endAmbiguity() { } } +func (p *parser) newCommentsFromTrivia(leadingTrivia, trailingTrivia []lexer.Trivia) ast.Comments { + comments := ast.Comments{ + Leading: []ast.Comment{}, + Trailing: []ast.Comment{}, + } + + for _, t := range leadingTrivia { + if isTriviaComment(t) { + comments.Leading = append(comments.Leading, ast.NewComment(p.memoryGauge, p.triviaSource(t))) + } + } + + for _, t := range trailingTrivia { + if isTriviaComment(t) { + comments.Trailing = append(comments.Trailing, ast.NewComment(p.memoryGauge, p.triviaSource(t))) + } + } + + return comments +} + +func isTriviaComment(t lexer.Trivia) bool { + // Other trivia types (space, newlines) are not needed in further stages of the compiler + return t.Type == lexer.TriviaTypeInlineComment || t.Type == lexer.TriviaTypeMultiLineComment +} + func ParseExpression( memoryGauge common.MemoryGauge, input []byte, diff --git a/runtime/parser/declaration_test.go b/runtime/parser/declaration_test.go index b2c47abda1..567a482eca 100644 --- a/runtime/parser/declaration_test.go +++ b/runtime/parser/declaration_test.go @@ -881,8 +881,12 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " Test", - StartPos: ast.Position{Line: 2, Column: 0, Offset: 9}, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// Test")), + }, + }, + StartPos: ast.Position{Line: 2, Column: 0, Offset: 9}, }, }, result, @@ -919,8 +923,13 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " First line\n Second line", - StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// First line")), + ast.NewComment(nil, []byte("/// Second line")), + }, + }, + StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, }, }, result, @@ -957,8 +966,12 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " Cool dogs.\n\n Cool cats!! ", - StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/** Cool dogs.\n\n Cool cats!! */")), + }, + }, + StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, }, }, result, @@ -1368,7 +1381,10 @@ func TestParseFunctionDeclaration(t *testing.T) { PreConditions: (*ast.Conditions)(nil), PostConditions: (*ast.Conditions)(nil), }, - DocString: "", + Comments: ast.Comments{ + Leading: []ast.Comment{}, + Trailing: []ast.Comment{}, + }, Identifier: ast.Identifier{ Identifier: "foo", Pos: ast.Position{ @@ -3511,7 +3527,10 @@ func TestParseCompositeDeclaration(t *testing.T) { }, }, }, - DocString: "", + Comments: ast.Comments{ + Leading: []ast.Comment{}, + Trailing: []ast.Comment{}, + }, Identifier: ast.Identifier{ Identifier: "getFoo", Pos: ast.Position{ @@ -8889,8 +8908,12 @@ func TestParseMemberDocStrings(t *testing.T) { Members: ast.NewUnmeteredMembers( []ast.Declaration{ &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " noReturnNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// noReturnNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "noReturnNoBlock", Pos: ast.Position{Offset: 78, Line: 5, Column: 18}, @@ -8904,8 +8927,12 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 74, Line: 5, Column: 14}, }, &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " returnNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// returnNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "returnNoBlock", Pos: ast.Position{Offset: 147, Line: 8, Column: 18}, @@ -8929,8 +8956,12 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 143, Line: 8, Column: 14}, }, &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " returnAndBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// returnAndBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "returnAndBlock", Pos: ast.Position{Offset: 220, Line: 11, Column: 18}, @@ -9004,8 +9035,12 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindUnknown, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " unknown", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// unknown")), + }, + }, Identifier: ast.Identifier{ Identifier: "unknown", Pos: ast.Position{Offset: 66, Line: 5, Column: 14}, @@ -9022,8 +9057,12 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindInitializer, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " initNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// initNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "init", Pos: ast.Position{Offset: 121, Line: 8, Column: 14}, diff --git a/runtime/parser/function.go b/runtime/parser/function.go index a6028be802..e6e5cc0aae 100644 --- a/runtime/parser/function.go +++ b/runtime/parser/function.go @@ -328,8 +328,8 @@ func parseFunctionDeclaration( nativePos *ast.Position, docString string, ) (*ast.FunctionDeclaration, error) { - - startPos := ast.EarliestPosition(p.current.StartPos, accessPos, purityPos, staticPos, nativePos) + startToken := p.current + startPos := ast.EarliestPosition(startToken.StartPos, accessPos, purityPos, staticPos, nativePos) // Skip the `fun` keyword p.nextSemanticToken() @@ -360,7 +360,7 @@ func parseFunctionDeclaration( return nil, err } - return ast.NewFunctionDeclaration( + return ast.NewFunctionDeclarationWithComments( p.memoryGauge, access, purity, @@ -373,6 +373,7 @@ func parseFunctionDeclaration( functionBlock, startPos, docString, + p.newCommentsFromTrivia(startToken.LeadingTrivia, []lexer.Trivia{}), ), nil } diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index 7b69245ce2..9734b9c36e 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -74,6 +74,8 @@ type lexer struct { prev rune // canBackup indicates whether stepping back is allowed canBackup bool + // currentTrivia stores the current leading or trailing Trivia + currentTrivia []Trivia } var _ TokenStream = &lexer{} @@ -99,6 +101,9 @@ func (l *lexer) Next() Token { pos, pos, ), + LeadingTrivia: l.currentTrivia, + // EOF has no trailing Trivia + TrailingTrivia: []Trivia{}, } } @@ -188,6 +193,8 @@ func (l *lexer) run(state stateFn) { for state != nil { state = state(l) } + + l.updatePreviousTrailingTrivia() } // next decodes the next rune (UTF8 character) from the input string. @@ -254,6 +261,8 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co endPos := l.endPos() + l.updatePreviousTrailingTrivia() + token := Token{ Type: ty, SpaceOrError: spaceOrError, @@ -267,23 +276,116 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co endPos.column, ), ), + LeadingTrivia: l.currentTrivia, + // Trailing Trivia can't be determined, as it wasn't consumed yet at this point. + TrailingTrivia: []Trivia{}, } + // Mark as fully consumed + l.currentTrivia = []Trivia{} + l.tokens = append(l.tokens, token) l.tokenCount = len(l.tokens) if consume { - l.startOffset = l.endOffset + l.consume(endPos) + } +} - l.startPos = endPos - r, _ := utf8.DecodeRune(l.input[l.endOffset-1:]) +func (l *lexer) updatePreviousTrailingTrivia() { + if l.tokenCount == 0 { + return + } - if r == '\n' { - l.startPos.line++ - l.startPos.column = 0 + // Split the current trivia into trailing trivia of the previous token + // and leading trivia of the next token. + var trailingTrivia []Trivia + var leadingTrivia []Trivia + trailingTriviaEnded := false + for _, trivia := range l.currentTrivia { + if trivia.Type == TriviaTypeNewLine { + trailingTriviaEnded = true + } + if trailingTriviaEnded { + leadingTrivia = append(leadingTrivia, trivia) } else { - l.startPos.column++ + trailingTrivia = append(trailingTrivia, trivia) + } + } + + l.tokens[l.tokenCount-1].TrailingTrivia = trailingTrivia + l.currentTrivia = leadingTrivia +} + +func (l *lexer) emitTrivia(triviaType TriviaType) { + endPos := l.endPos() + + currentRange := ast.NewRange( + l.memoryGauge, + l.startPosition(), + ast.NewPosition( + l.memoryGauge, + l.endOffset-1, + endPos.line, + endPos.column, + ), + ) + + if l.currentTrivia == nil { + l.currentTrivia = []Trivia{} + } + + l.currentTrivia = append(l.currentTrivia, Trivia{ + Type: triviaType, + Range: currentRange, + }) + + l.consume(endPos) +} + +const useLegacyTrivia = false + +// TODO(preserve-comments): Remove after refactoring is complete +func (l *lexer) emitLegacyTrivia(legacyTriviaType TokenType) { + if !useLegacyTrivia { + return + } + + if legacyTriviaType == TokenSpace { + trivia := l.input[l.startOffset:l.endOffset] + + var containsNewline bool + for _, r := range trivia { + if r == '\n' { + containsNewline = true + } } + + l.emit( + TokenSpace, + Space{ + ContainsNewline: containsNewline, + }, + l.startPosition(), + true, + ) + } else { + l.emitType(legacyTriviaType) + } +} + +// endPos pre-computed end-position by calling l.endPos() +func (l *lexer) consume(endPos position) { + l.startOffset = l.endOffset + + l.startPos = endPos + r, _ := utf8.DecodeRune(l.input[l.endOffset-1:]) + + if r == '\n' { + l.startPos.line++ + l.startPos.column = 0 + } else { + l.startPos.column++ } } @@ -337,16 +439,13 @@ func (l *lexer) emitError(err error) { l.emit(TokenError, err, rangeStart, false) } -func (l *lexer) scanSpace() (containsNewline bool) { +func (l *lexer) scanSpace() { // lookahead is already lexed. // parse more, if any l.acceptWhile(func(r rune) bool { switch r { case ' ', '\t', '\r': return true - case '\n': - containsNewline = true - return true default: return false } diff --git a/runtime/parser/lexer/lexer_test.go b/runtime/parser/lexer/lexer_test.go index 51f8f53f34..d46a87156e 100644 --- a/runtime/parser/lexer/lexer_test.go +++ b/runtime/parser/lexer/lexer_test.go @@ -88,19 +88,6 @@ func TestLexBasic(t *testing.T) { testLex(t, " 01\t 10", []token{ - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - }, - }, - Source: " ", - }, { Token: Token{ Type: TokenDecimalIntegerLiteral, @@ -108,21 +95,26 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, }, - }, - Source: "01", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + }, + }, + }, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, + }, + }, }, }, - Source: "\t ", + Source: "01", }, { Token: Token{ @@ -215,21 +207,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, }, - }, - Source: "2", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + }, + }, }, }, - Source: " ", + Source: "2", }, { Token: Token{ @@ -238,21 +226,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, - }, - Source: "+", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + }, + }, }, }, - Source: " ", + Source: "+", }, { Token: Token{ @@ -271,21 +255,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 6, Offset: 6}, EndPos: ast.Position{Line: 1, Column: 6, Offset: 6}, }, - }, - Source: ")", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 7, Offset: 7}, - EndPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + EndPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + }, + }, }, }, - Source: " ", + Source: ")", }, { Token: Token{ @@ -294,21 +274,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 8, Offset: 8}, EndPos: ast.Position{Line: 1, Column: 8, Offset: 8}, }, - }, - Source: "*", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 9, Offset: 9}, - EndPos: ast.Position{Line: 1, Column: 9, Offset: 9}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 9, Offset: 9}, + EndPos: ast.Position{Line: 1, Column: 9, Offset: 9}, + }, + }, }, }, - Source: " ", + Source: "*", }, { Token: Token{ @@ -354,21 +330,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, }, - }, - Source: "2", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + }, + }, }, }, - Source: " ", + Source: "2", }, { Token: Token{ @@ -377,21 +349,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, - }, - Source: "-", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + }, + }, }, }, - Source: " ", + Source: "-", }, { Token: Token{ @@ -410,21 +378,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 6, Offset: 6}, EndPos: ast.Position{Line: 1, Column: 6, Offset: 6}, }, - }, - Source: ")", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 7, Offset: 7}, - EndPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + EndPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + }, + }, }, }, - Source: " ", + Source: ")", }, { Token: Token{ @@ -433,21 +397,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 8, Offset: 8}, EndPos: ast.Position{Line: 1, Column: 8, Offset: 8}, }, - }, - Source: "/", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 9, Offset: 9}, - EndPos: ast.Position{Line: 1, Column: 9, Offset: 9}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 9, Offset: 9}, + EndPos: ast.Position{Line: 1, Column: 9, Offset: 9}, + }, + }, }, }, - Source: " ", + Source: "/", }, { Token: Token{ @@ -483,21 +443,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, }, - }, - Source: "1", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: true, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 2, Column: 1, Offset: 4}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, }, }, - Source: " \n ", + Source: "1", }, { Token: Token{ @@ -506,21 +462,24 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 2, Offset: 5}, EndPos: ast.Position{Line: 2, Column: 2, Offset: 5}, }, - }, - Source: "2", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: true, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 2, Column: 3, Offset: 6}, - EndPos: ast.Position{Line: 2, Column: 3, Offset: 6}, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeNewLine, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + }, + }, + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 2, Column: 0, Offset: 3}, + EndPos: ast.Position{Line: 2, Column: 1, Offset: 4}, + }, + }, }, }, - Source: "\n", + Source: "2", }, { Token: Token{ @@ -529,6 +488,15 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 3, Column: 0, Offset: 7}, EndPos: ast.Position{Line: 3, Column: 0, Offset: 7}, }, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeNewLine, + Range: ast.Range{ + StartPos: ast.Position{Line: 2, Column: 3, Offset: 6}, + EndPos: ast.Position{Line: 2, Column: 3, Offset: 6}, + }, + }, + }, }, }, }, @@ -546,21 +514,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, }, - }, - Source: "1", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, }, }, - Source: " ", + Source: "1", }, { Token: Token{ @@ -569,21 +533,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, - }, - Source: "??", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + }, + }, }, }, - Source: " ", + Source: "??", }, { Token: Token{ @@ -873,6 +833,182 @@ func TestLexBasic(t *testing.T) { }, ) }) + + t.Run("trivia", func(t *testing.T) { + testLex(t, + "// test is in next line \n/* test is here */ test // test is in the same line\n// test is in previous line", + []token{ + { + Token: Token{ + Type: TokenIdentifier, + Range: ast.Range{ + StartPos: ast.Position{Line: 2, Column: 19, Offset: 44}, + EndPos: ast.Position{Line: 2, Column: 22, Offset: 47}, + }, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeInlineComment, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 0, + Line: 1, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 23, + Line: 1, + Column: 23, + }, + }, + }, + { + Type: TriviaTypeNewLine, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 24, + Line: 1, + Column: 24, + }, + EndPos: ast.Position{ + Offset: 24, + Line: 1, + Column: 24, + }, + }, + }, + { + Type: TriviaTypeMultiLineComment, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 25, + Line: 2, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 42, + Line: 2, + Column: 17, + }, + }, + }, + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 43, + Line: 2, + Column: 18, + }, + EndPos: ast.Position{ + Offset: 43, + Line: 2, + Column: 18, + }, + }, + }, + }, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 48, + Line: 2, + Column: 23, + }, + EndPos: ast.Position{ + Offset: 48, + Line: 2, + Column: 23, + }, + }, + }, + { + Type: TriviaTypeInlineComment, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 49, + Line: 2, + Column: 24, + }, + EndPos: ast.Position{ + Offset: 75, + Line: 2, + Column: 50, + }, + }, + }, + }, + }, + Source: "test", + }, + { + Token: Token{ + Type: TokenEOF, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 27, Offset: 104}, + EndPos: ast.Position{Line: 3, Column: 27, Offset: 104}, + }, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeNewLine, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 76, + Line: 2, + Column: 51, + }, + EndPos: ast.Position{ + Offset: 76, + Line: 2, + Column: 51, + }, + }, + }, + { + Type: TriviaTypeInlineComment, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 77, + Line: 3, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 103, + Line: 3, + Column: 26, + }, + }, + }, + }, + TrailingTrivia: []Trivia{}, + }, + }, + }, + ) + }) +} + +func TestLexTrivia(t *testing.T) { + + t.Parallel() + + // TODO(preserve-comments): Trailing Trivia contains one character more than it should + t.Run("Trivia", func(t *testing.T) { + testLex(t, ` +// This transaction calculates sum +transaction ( + // First operand 1 + a: Int, // First operand 2 + // Second operand 1 + b: Int // Second operand 2 +) { + // Logs the sum 1 + log(a, b) // Logs the sum 2 +} + +`, []token{}) // TODO(preserve-comments): Define expected output + }) } func TestLexString(t *testing.T) { @@ -1028,19 +1164,6 @@ func TestLexString(t *testing.T) { }, Source: "\"", }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: true, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, - Source: "\n", - }, { Token: Token{ Type: TokenEOF, @@ -1048,6 +1171,15 @@ func TestLexString(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 0, Offset: 2}, EndPos: ast.Position{Line: 2, Column: 0, Offset: 2}, }, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeNewLine, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, + }, }, }, }, @@ -1068,19 +1200,6 @@ func TestLexString(t *testing.T) { }, Source: "\"te", }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: true, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - }, - }, - Source: "\n", - }, { Token: Token{ Type: TokenEOF, @@ -1088,6 +1207,15 @@ func TestLexString(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 0, Offset: 4}, EndPos: ast.Position{Line: 2, Column: 0, Offset: 4}, }, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeNewLine, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + }, + }, + }, }, }, }, @@ -1162,19 +1290,6 @@ func TestLexString(t *testing.T) { }, Source: "\"\\", }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: true, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - }, - }, - Source: "\n", - }, { Token: Token{ Type: TokenEOF, @@ -1182,6 +1297,15 @@ func TestLexString(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 0, Offset: 3}, EndPos: ast.Position{Line: 2, Column: 0, Offset: 3}, }, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeNewLine, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + }, + }, + }, }, }, }, @@ -1199,53 +1323,14 @@ func TestLexBlockComment(t *testing.T) { []token{ { Token: Token{ - Type: TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, - Source: "/*", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 9, Offset: 9}, - }, - }, - Source: ` // *X `, - }, - { - Token: Token{ - Type: TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 10, Offset: 10}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - }, - }, - Source: "/*", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 12, Offset: 12}, - EndPos: ast.Position{Line: 1, Column: 17, Offset: 17}, - }, - }, - Source: ` \\* `, - }, - { - Token: Token{ - Type: TokenBlockCommentEnd, + Type: TokenError, + SpaceOrError: errors.New(`end of the file in a comment`), Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 18, Offset: 18}, - EndPos: ast.Position{Line: 1, Column: 19, Offset: 19}, + StartPos: ast.Position{Line: 1, Column: 20, Offset: 20}, + EndPos: ast.Position{Line: 1, Column: 20, Offset: 20}, }, }, - Source: "*/", + Source: "\000", }, { Token: Token{ @@ -1264,89 +1349,6 @@ func TestLexBlockComment(t *testing.T) { testLex(t, `/* test foo /* bar */ asd */ `, []token{ - { - Token: Token{ - Type: TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, - Source: "/*", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - }, - }, - Source: ` test foo `, - }, - { - Token: Token{ - Type: TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 12, Offset: 12}, - EndPos: ast.Position{Line: 1, Column: 13, Offset: 13}, - }, - }, - Source: "/*", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 14, Offset: 14}, - EndPos: ast.Position{Line: 1, Column: 18, Offset: 18}, - }, - }, - Source: ` bar `, - }, - { - Token: Token{ - Type: TokenBlockCommentEnd, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 19, Offset: 19}, - EndPos: ast.Position{Line: 1, Column: 20, Offset: 20}, - }, - }, - Source: "*/", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 21, Offset: 21}, - EndPos: ast.Position{Line: 1, Column: 25, Offset: 25}, - }, - }, - Source: ` asd `, - }, - { - Token: Token{ - Type: TokenBlockCommentEnd, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 26, Offset: 26}, - EndPos: ast.Position{Line: 1, Column: 27, Offset: 27}, - }, - }, - Source: "*/", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 28, Offset: 28}, - EndPos: ast.Position{Line: 1, Column: 29, Offset: 29}, - }, - }, - Source: " ", - }, { Token: Token{ Type: TokenEOF, @@ -1354,6 +1356,22 @@ func TestLexBlockComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 30, Offset: 30}, EndPos: ast.Position{Line: 1, Column: 30, Offset: 30}, }, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeMultiLineComment, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 1, Column: 27, Offset: 27}, + }, + }, + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 28, Offset: 28}, + EndPos: ast.Position{Line: 1, Column: 29, Offset: 29}, + }, + }, + }, }, }, }, @@ -1979,19 +1997,6 @@ func TestLexIntegerLiterals(t *testing.T) { }, Source: "0", }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: true, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, - Source: "\n", - }, { Token: Token{ Type: TokenEOF, @@ -1999,6 +2004,15 @@ func TestLexIntegerLiterals(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 0, Offset: 2}, EndPos: ast.Position{Line: 2, Column: 0, Offset: 2}, }, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeNewLine, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, + }, }, }, }, @@ -2233,19 +2247,6 @@ func TestLexLineComment(t *testing.T) { testLex(t, ` foo // bar `, []token{ - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - }, - }, - Source: " ", - }, { Token: Token{ Type: TokenIdentifier, @@ -2253,31 +2254,33 @@ func TestLexLineComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, - }, - Source: "foo", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - }, - }, - Source: " ", - }, - { - Token: Token{ - Type: TokenLineComment, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + }, + }, + }, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + }, + }, + { + Type: TriviaTypeInlineComment, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, + EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, + }, + }, }, }, - Source: "// bar ", + Source: "foo", }, { Token: Token{ @@ -2298,19 +2301,6 @@ func TestLexLineComment(t *testing.T) { t, " foo // bar \n baz", []token{ - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - }, - }, - Source: " ", - }, { Token: Token{ Type: TokenIdentifier, @@ -2318,44 +2308,33 @@ func TestLexLineComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, - }, - Source: "foo", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + }, + }, + }, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + }, + }, + { + Type: TriviaTypeInlineComment, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, + EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, + }, + }, }, }, - Source: " ", - }, - { - Token: Token{ - Type: TokenLineComment, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - }, - }, - Source: "// bar ", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: true, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 12, Offset: 12}, - EndPos: ast.Position{Line: 2, Column: 0, Offset: 13}, - }, - }, - Source: "\n ", + Source: "foo", }, { Token: Token{ @@ -2364,6 +2343,22 @@ func TestLexLineComment(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 1, Offset: 14}, EndPos: ast.Position{Line: 2, Column: 3, Offset: 16}, }, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeNewLine, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 12, Offset: 12}, + EndPos: ast.Position{Line: 1, Column: 12, Offset: 12}, + }, + }, + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 2, Column: 0, Offset: 13}, + EndPos: ast.Position{Line: 2, Column: 0, Offset: 13}, + }, + }, + }, }, Source: "baz", }, @@ -2396,19 +2391,14 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, }, - }, - tokenStream.Next(), - ) - - assert.Equal(t, - Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, }, }, tokenStream.Next(), @@ -2423,19 +2413,14 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, }, - }, - tokenStream.Next(), - ) - - assert.Equal(t, - Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + }, + }, }, }, tokenStream.Next(), @@ -2462,6 +2447,7 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, }, + TrailingTrivia: []Trivia{}, }, tokenStream.Next(), ) @@ -2473,6 +2459,7 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, }, + TrailingTrivia: []Trivia{}, }, tokenStream.Next(), ) @@ -2490,19 +2477,14 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, }, - }, - tokenStream.Next(), - ) - - assert.Equal(t, - Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + }, + }, }, }, tokenStream.Next(), @@ -2529,6 +2511,7 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, }, + TrailingTrivia: []Trivia{}, }, tokenStream.Next(), ) @@ -2540,6 +2523,7 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, }, + TrailingTrivia: []Trivia{}, }, tokenStream.Next(), ) @@ -2561,19 +2545,14 @@ func TestEOFsAfterError(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, }, - }, - tokenStream.Next(), - ) - - assert.Equal(t, - Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, }, }, tokenStream.Next(), @@ -2603,6 +2582,7 @@ func TestEOFsAfterError(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, }, + TrailingTrivia: []Trivia{}, }, tokenStream.Next(), ) @@ -2627,6 +2607,7 @@ func TestEOFsAfterEmptyInput(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, }, + TrailingTrivia: []Trivia{}, }, tokenStream.Next(), ) @@ -2638,8 +2619,9 @@ func TestLimit(t *testing.T) { t.Parallel() var b strings.Builder + // TODO(preserve-comments): Token limit is affected, because comments are not treated as separate tokens for i := 0; i < 300000; i++ { - b.WriteString("x ") + b.WriteString("x x ") } code := b.String() diff --git a/runtime/parser/lexer/state.go b/runtime/parser/lexer/state.go index 4b252d6f09..315f7afa29 100644 --- a/runtime/parser/lexer/state.go +++ b/runtime/parser/lexer/state.go @@ -111,9 +111,11 @@ func rootState(l *lexer) stateFn { case '_': return identifierState case ' ', '\t', '\r': - return spaceState(false) + return spaceState case '\n': - return spaceState(true) + l.emitLegacyTrivia(TokenSpace) + l.emitTrivia(TriviaTypeNewLine) + return rootState case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': return numberState case '"': @@ -121,11 +123,12 @@ func rootState(l *lexer) stateFn { case '/': r = l.next() switch r { + // TODO(preserve-comments): Deprecate Trivia token types case '/': return lineCommentState case '*': - l.emitType(TokenBlockCommentStart) - return blockCommentState(0) + l.emitLegacyTrivia(TokenBlockCommentStart) + return blockCommentState(l, 0) default: l.backupOne() l.emitType(TokenSlash) @@ -260,23 +263,16 @@ type Space struct { ContainsNewline bool } -func spaceState(startIsNewline bool) stateFn { - return func(l *lexer) stateFn { - containsNewline := l.scanSpace() - containsNewline = containsNewline || startIsNewline - - common.UseMemory(l.memoryGauge, common.SpaceTokenMemoryUsage) - - l.emit( - TokenSpace, - Space{ - ContainsNewline: containsNewline, - }, - l.startPosition(), - true, - ) - return rootState - } +func spaceState(l *lexer) stateFn { + l.scanSpace() + + // TODO(preserve-comments): Do we need to track memory for other token types as well? + common.UseMemory(l.memoryGauge, common.SpaceTokenMemoryUsage) + + l.emitLegacyTrivia(TokenSpace) + l.emitTrivia(TriviaTypeSpace) + + return rootState } func identifierState(l *lexer) stateFn { @@ -307,12 +303,14 @@ func stringState(l *lexer) stateFn { func lineCommentState(l *lexer) stateFn { l.scanLineComment() - l.emitType(TokenLineComment) + l.emitLegacyTrivia(TokenLineComment) + l.emitTrivia(TriviaTypeInlineComment) return rootState } -func blockCommentState(nesting int) stateFn { +func blockCommentState(l *lexer, nesting int) stateFn { if nesting < 0 { + l.emitTrivia(TriviaTypeMultiLineComment) return rootState } @@ -320,16 +318,17 @@ func blockCommentState(nesting int) stateFn { r := l.next() switch r { case EOF: + l.emitError(fmt.Errorf("end of the file in a comment")) return nil case '/': beforeSlashOffset := l.prevEndOffset if l.acceptOne('*') { starOffset := l.endOffset l.endOffset = beforeSlashOffset - l.emitType(TokenBlockCommentContent) + l.emitLegacyTrivia(TokenBlockCommentContent) l.endOffset = starOffset - l.emitType(TokenBlockCommentStart) - return blockCommentState(nesting + 1) + l.emitLegacyTrivia(TokenBlockCommentStart) + return blockCommentState(l, nesting+1) } case '*': @@ -337,13 +336,13 @@ func blockCommentState(nesting int) stateFn { if l.acceptOne('/') { slashOffset := l.endOffset l.endOffset = beforeStarOffset - l.emitType(TokenBlockCommentContent) + l.emitLegacyTrivia(TokenBlockCommentContent) l.endOffset = slashOffset - l.emitType(TokenBlockCommentEnd) - return blockCommentState(nesting - 1) + l.emitLegacyTrivia(TokenBlockCommentEnd) + return blockCommentState(l, nesting-1) } } - return blockCommentState(nesting) + return blockCommentState(l, nesting) } } diff --git a/runtime/parser/lexer/token.go b/runtime/parser/lexer/token.go index cfda861bdf..9982a3d616 100644 --- a/runtime/parser/lexer/token.go +++ b/runtime/parser/lexer/token.go @@ -23,9 +23,14 @@ import ( ) type Token struct { + // TODO(preserve-comments): Rename to Error error SpaceOrError any ast.Range Type TokenType + // leading Trivia up to and including the first contiguous sequence of newlines characters. + LeadingTrivia []Trivia + // trailing Trivia up to, but not including, the next newline character. + TrailingTrivia []Trivia } func (t Token) Is(ty TokenType) bool { @@ -33,7 +38,5 @@ func (t Token) Is(ty TokenType) bool { } func (t Token) Source(input []byte) []byte { - startOffset := t.StartPos.Offset - endOffset := t.EndPos.Offset + 1 - return input[startOffset:endOffset] + return t.Range.Source(input) } diff --git a/runtime/parser/lexer/tokentype.go b/runtime/parser/lexer/tokentype.go index 0a15c19b6f..dd5ced009c 100644 --- a/runtime/parser/lexer/tokentype.go +++ b/runtime/parser/lexer/tokentype.go @@ -69,6 +69,7 @@ const ( TokenEqualEqual TokenExclamationMark TokenNotEqual + // TODO(preserve-comments): Deprecate Trivia token types TokenBlockCommentStart TokenBlockCommentEnd TokenBlockCommentContent diff --git a/runtime/parser/lexer/trivia.go b/runtime/parser/lexer/trivia.go new file mode 100644 index 0000000000..4eb0dfa0e9 --- /dev/null +++ b/runtime/parser/lexer/trivia.go @@ -0,0 +1,19 @@ +package lexer + +import "github.com/onflow/cadence/runtime/ast" + +type Trivia struct { + Type TriviaType + // Position within the source code (includes opening/closing comment characters in case of comment trivia type) + ast.Range +} + +type TriviaType uint8 + +const ( + TriviaTypeUnknown TriviaType = iota + TriviaTypeInlineComment + TriviaTypeMultiLineComment + TriviaTypeNewLine + TriviaTypeSpace +) diff --git a/runtime/parser/parser.go b/runtime/parser/parser.go index 497d1f54e4..6e5b64910a 100644 --- a/runtime/parser/parser.go +++ b/runtime/parser/parser.go @@ -286,6 +286,11 @@ func (p *parser) tokenSource(token lexer.Token) []byte { return token.Source(input) } +func (p *parser) triviaSource(trivia lexer.Trivia) []byte { + input := p.tokens.Input() + return trivia.Source(input) +} + func (p *parser) currentTokenSource() []byte { return p.tokenSource(p.current) } @@ -553,6 +558,32 @@ func (p *parser) tokenToIdentifier(token lexer.Token) ast.Identifier { ) } +func (p *parser) newCommentsFromTrivia(leadingTrivia, trailingTrivia []lexer.Trivia) ast.Comments { + comments := ast.Comments{ + Leading: []ast.Comment{}, + Trailing: []ast.Comment{}, + } + + for _, t := range leadingTrivia { + if isTriviaComment(t) { + comments.Leading = append(comments.Leading, ast.NewComment(p.memoryGauge, p.triviaSource(t))) + } + } + + for _, t := range trailingTrivia { + if isTriviaComment(t) { + comments.Trailing = append(comments.Trailing, ast.NewComment(p.memoryGauge, p.triviaSource(t))) + } + } + + return comments +} + +func isTriviaComment(t lexer.Trivia) bool { + // Other trivia types (space, newlines) are not needed in further stages of the compiler + return t.Type == lexer.TriviaTypeInlineComment || t.Type == lexer.TriviaTypeMultiLineComment +} + func (p *parser) startAmbiguity() { if p.ambiguityLevel == 0 { p.localReplayedTokensCount = 0 diff --git a/runtime/parser/parser_test.go b/runtime/parser/parser_test.go index ff816dd4c5..a81ffaa1af 100644 --- a/runtime/parser/parser_test.go +++ b/runtime/parser/parser_test.go @@ -1013,3 +1013,68 @@ func TestParseWhitespaceAtEnd(t *testing.T) { assert.Empty(t, errs) } + +func TestParseTrivia(t *testing.T) { + + t.Parallel() + + t.Run("function declaration", func(t *testing.T) { + res, errs := ParseProgram( + nil, + []byte(` +/// Inline doc 1 of first +/// Inline doc 2 of first +fun first() {} // Trailing inline comment of first + +/** +Multi-line doc 1 of second +*/ +/** +Multi-line doc 2 of second +*/ +fun second() {} /** +Trailing multi-line comment of second +*/ +`), + Config{}, + ) + + assert.Empty(t, errs) + assert.NotNil(t, res) + + first, ok := res.Declarations()[0].(*ast.FunctionDeclaration) + assert.True(t, ok) + assert.Equal(t, " Inline doc 1 of first\n Inline doc 2 of first", first.DeclarationDocString()) + assert.Equal(t, ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// Inline doc 1 of first")), + ast.NewComment(nil, []byte("/// Inline doc 2 of first")), + }, + Trailing: []ast.Comment{}, + }, first.Comments) + assert.Equal(t, ast.Comments{ + Leading: []ast.Comment{}, + Trailing: []ast.Comment{ + ast.NewComment(nil, []byte("// Trailing inline comment of first")), + }, + }, first.FunctionBlock.Block.Comments) + + second, ok := res.Declarations()[1].(*ast.FunctionDeclaration) + assert.True(t, ok) + assert.Equal(t, "\nMulti-line doc 1 of second\n\n\nMulti-line doc 2 of second\n", second.DeclarationDocString()) + assert.Equal(t, ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/**\nMulti-line doc 1 of second\n*/")), + ast.NewComment(nil, []byte("/**\nMulti-line doc 2 of second\n*/")), + }, + Trailing: []ast.Comment{}, + }, second.Comments) + assert.Equal(t, ast.Comments{ + Leading: []ast.Comment{}, + Trailing: []ast.Comment{ + ast.NewComment(nil, []byte("/**\nTrailing multi-line comment of second\n*/")), + }, + }, second.FunctionBlock.Block.Comments) + }) + +} diff --git a/runtime/parser/statement.go b/runtime/parser/statement.go index cd5c9b93e4..35deb28838 100644 --- a/runtime/parser/statement.go +++ b/runtime/parser/statement.go @@ -556,7 +556,7 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { return ast.NewFunctionBlock( p.memoryGauge, - ast.NewBlock( + ast.NewBlockWithComments( p.memoryGauge, statements, ast.NewRange( @@ -564,6 +564,7 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { startToken.StartPos, endToken.EndPos, ), + p.newCommentsFromTrivia(startToken.LeadingTrivia, endToken.TrailingTrivia), ), preConditions, postConditions, diff --git a/runtime/sema/check_composite_declaration.go b/runtime/sema/check_composite_declaration.go index e6a42aa358..d2e4d45944 100644 --- a/runtime/sema/check_composite_declaration.go +++ b/runtime/sema/check_composite_declaration.go @@ -1845,7 +1845,7 @@ func (checker *Checker) defaultMembersAndOrigins( TypeAnnotation: fieldTypeAnnotation, VariableKind: ast.VariableKindConstant, ArgumentLabels: argumentLabels, - DocString: function.DocString, + DocString: function.DeclarationDocString(), HasImplementation: hasImplementation, HasConditions: hasConditions, }) diff --git a/runtime/sema/check_function.go b/runtime/sema/check_function.go index 4a716a204a..3ef555f3d2 100644 --- a/runtime/sema/check_function.go +++ b/runtime/sema/check_function.go @@ -147,7 +147,7 @@ func (checker *Checker) declareFunctionDeclaration( _, err := checker.valueActivations.declare(variableDeclaration{ identifier: declaration.Identifier.Identifier, ty: functionType, - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), access: checker.accessFromAstAccess(declaration.Access), kind: common.DeclarationKindFunction, pos: declaration.Identifier.Pos, diff --git a/runtime/sema/positioninfo.go b/runtime/sema/positioninfo.go index 8c4b19dc5e..9b0a8a5687 100644 --- a/runtime/sema/positioninfo.go +++ b/runtime/sema/positioninfo.go @@ -111,7 +111,7 @@ func (i *PositionInfo) recordFunctionDeclarationOrigin( DeclarationKind: common.DeclarationKindFunction, StartPos: &startPosition, EndPos: &endPosition, - DocString: function.DocString, + DocString: function.DeclarationDocString(), } i.Occurrences.Put(