allow comment in between tokens of a field definition ( #131 ) (#142)

* allow comment in between tokens of a field definition

* also read comment in between sequence and options or comma

* applied suggestions from PR

Author: emicklei

comment_test.go

@@ -310,3 +310,29 @@ message Foo {
 		t.Fatalf("got [%v] want [%v]", got, want)
 	}
 }
+
+func TestNormalFieldInlineComment(t *testing.T) {
+	src := `message Example {
+		/* i'm */ optional /* a comment */ bool /* too */ field /* hard */ = /* to */ 1 /* read */;
+	}`
+	p := newParserOn(src)
+	def, err := p.Parse()
+	if err != nil {
+		t.Fatal(err)
+	}
+	m := def.Elements[0].(*Message)
+	f := m.Elements[1].(*NormalField)
+	lines := f.Field.InlineComment.Lines
+	if got, want := len(lines), 5; got != want {
+		t.Fatalf("got [%v:%T] want [%v:%T]", got, got, want, want)
+	}
+	if got, want := lines[0], " a comment "; got != want {
+		t.Errorf("got [%v:%T] want [%v:%T]", got, got, want, want)
+	}
+	if got, want := lines[1], " too "; got != want {
+		t.Errorf("got [%v:%T] want [%v:%T]", got, got, want, want)
+	}
+	if got, want := lines[4], " read "; got != want {
+		t.Errorf("got [%v:%T] want [%v:%T]", got, got, want, want)
+	}
+}

field.go

@@ -68,8 +68,15 @@ func (f *NormalField) Doc() *Comment {
 // [ "repeated" | "optional" ] type fieldName "=" fieldNumber [ "[" fieldOptions "]" ] ";"
 func (f *NormalField) parse(p *Parser) error {
 	for {
-		_, tok, lit := p.nextTypeName()
+		pos, tok, lit := p.nextTypeName()
 		switch tok {
+		case tCOMMENT:
+			c := newComment(pos, lit)
+			if f.InlineComment == nil {
+				f.InlineComment = c
+			} else {
+				f.InlineComment.Merge(c)
+			}
 		case tREPEATED:
 			f.Repeated = true
 			return f.parse(p)
@@ -90,24 +97,53 @@ done:
 // parseFieldAfterType expects:
 // fieldName "=" fieldNumber [ "[" fieldOptions "]" ] ";
 func parseFieldAfterType(f *Field, p *Parser, parent Visitee) error {
-	pos, tok, lit := p.next()
-	if tok != tIDENT {
-		if !isKeyword(tok) {
-			return p.unexpected(lit, "field identifier", f)
+	expectedToken := tIDENT
+	expected := "field identifier"
+
+	for {
+		pos, tok, lit := p.next()
+		if tok == tCOMMENT {
+			c := newComment(pos, lit)
+			if f.InlineComment == nil {
+				f.InlineComment = c
+			} else {
+				f.InlineComment.Merge(c)
+			}
+			continue
+		}
+		if tok != expectedToken {
+			return p.unexpected(lit, expected, f)
+		}
+		// found expected token
+		if tok == tIDENT {
+			if isKeyword(tok) {
+				return p.unexpected(lit, expected, f)
+			}
+			f.Name = lit
+			expectedToken = tEQUALS
+			expected = "field ="
+			continue
+		}
+		if tok == tEQUALS {
+			expectedToken = tNUMBER
+			expected = "field sequence number"
+			continue
+		}
+		if tok == tNUMBER {
+			// put it back so we can use the generic nextInteger
+			p.nextPut(pos, tok, lit)
+			i, err := p.nextInteger()
+			if err != nil {
+				return p.unexpected(lit, expected, f)
+			}
+			f.Sequence = i
+			break
 		}
 	}
-	f.Name = lit
-	pos, tok, lit = p.next()
-	if tok != tEQUALS {
-		return p.unexpected(lit, "field =", f)
-	}
-	i, err := p.nextInteger()
-	if err != nil {
-		return p.unexpected(lit, "field sequence number", f)
-	}
-	f.Sequence = i
+	consumeFieldComments(f, p)
+
 	// see if there are options
-	pos, tok, _ = p.next()
+	pos, tok, lit := p.next()
 	if tLEFTSQUARE != tok {
 		p.nextPut(pos, tok, lit)
 		return nil
@@ -135,6 +171,21 @@ func parseFieldAfterType(f *Field, p *Parser, parent Visitee) error {
 	return nil
 }
 
+func consumeFieldComments(f *Field, p *Parser) {
+	pos, tok, lit := p.next()
+	for tok == tCOMMENT {
+		c := newComment(pos, lit)
+		if f.InlineComment == nil {
+			f.InlineComment = c
+		} else {
+			f.InlineComment.Merge(c)
+		}
+		pos, tok, lit = p.next()
+	}
+	// no longer a comment, put it back
+	p.nextPut(pos, tok, lit)
+}
+
 // MapField represents a map entry in a message.
 type MapField struct {
 	*Field

parser.go

@@ -190,7 +190,7 @@ func (p *Parser) nextInteger() (i int, err error) {
 		i, err = p.nextInteger()
 		return i * -1, err
 	}
-	if tok != tIDENT {
+	if tok != tNUMBER {
 		return 0, errors.New("non integer")
 	}
 	if strings.HasPrefix(lit, "0x") || strings.HasPrefix(lit, "0X") {

token.go

@@ -24,6 +24,7 @@
 package proto
 
 import (
+	"strconv"
 	"strings"
 )
 
@@ -78,6 +79,9 @@ const (
 	tENUM
 	tSTREAM
 
+	// numbers (pos or neg, float)
+	tNUMBER
+
 	// BEGIN proto2
 	tOPTIONAL
 	tGROUP
@@ -119,6 +123,15 @@ func isComment(lit string) bool {
 	return strings.HasPrefix(lit, "//") || strings.HasPrefix(lit, "/*")
 }
 
+func isNumber(lit string) bool {
+	if strings.HasPrefix(lit, "0x") || strings.HasPrefix(lit, "0X") {
+		_, err := strconv.ParseInt(lit, 0, 64)
+		return err == nil
+	}
+	_, err := strconv.ParseFloat(lit, 64)
+	return err == nil
+}
+
 const doubleQuoteRune = rune('"')
 
 // unQuote removes one matching leading and trailing single or double quote.
@@ -220,6 +233,9 @@ func asToken(literal string) token {
 		return tREQUIRED
 	default:
 		// special cases
+		if isNumber(literal) {
+			return tNUMBER
+		}
 		if isComment(literal) {
 			return tCOMMENT
 		}

token_test.go

@@ -51,3 +51,22 @@ func TestUnQuoteCases(t *testing.T) {
 		}
 	}
 }
+
+func TestIsNumber(t *testing.T) {
+	for i, each := range []struct {
+		input    string
+		isNumber bool
+	}{
+		{`1`, true},
+		{`1.2`, true},
+		{`-1.02`, true},
+		{`a1`, false},
+		{`0x12`, true},
+		{`0X77777`, true},
+	} {
+		got := isNumber(each.input)
+		if got != each.isNumber {
+			t.Errorf("[%d] got [%v] want [%v]", i, got, each.isNumber)
+		}
+	}
+}