From 47d11b829c3de63a55cf90624752838ac4e4d80b Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Wed, 2 Oct 2019 17:19:13 -0700 Subject: [PATCH 01/22] fixed mutliline value handling --- ini_multiline_value_fix_test.go | 60 +++++++++++++++++++++++++++++++++ parser.go | 11 ++++-- testdata/multiline.ini | 9 +++++ 3 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 ini_multiline_value_fix_test.go create mode 100644 testdata/multiline.ini diff --git a/ini_multiline_value_fix_test.go b/ini_multiline_value_fix_test.go new file mode 100644 index 0000000..4d51e53 --- /dev/null +++ b/ini_multiline_value_fix_test.go @@ -0,0 +1,60 @@ +package ini_test + +import ( + "github.com/r3db34n/ini" + . "github.com/smartystreets/goconvey/convey" + "path/filepath" + "testing" +) + +type testData struct { + Value1 string `ini:"value1"` + Value2 string `ini:"value2"` +} + +func TestMultiline(t *testing.T) { + Value1 := "some text here\n"+ + "some more text here\n"+ + "\n"+ + "there is an empty line above and below\n" + + Value2 := "there is an empty line above\n"+ + "that is not indented so it should not be part\n"+ + "of the value" + + Convey("Parse Python-style multiline values", t, func() { + + // enable Python-style multiline values + opts := ini.LoadOptions{ + AllowPythonMultilineValues: true, + } + + // load test data + path := filepath.Join("testdata", "multiline.ini") + data, e := ini.LoadSources(opts, path) + + // Should have no error + So(e, ShouldBeNil) + + // Should have parsed data + So(data, ShouldNotBeNil) + + // Should have only the default section + So(len(data.Sections()), ShouldEqual, 1) + + // Should have default section + defaultSection := data.Section("") + So(defaultSection, ShouldNotBeNil) + + // Default section should map to test data struct + var testData testData + e = defaultSection.MapTo(&testData) + So(e, ShouldBeNil) + + // 'value1' should match expected value + So(testData.Value1, ShouldEqual, Value1) + + // 'value2' should match expected value + So(testData.Value2, ShouldEqual, Value2) + }) +} \ No newline at end of file diff --git a/parser.go b/parser.go index 7c22a25..9c8c70e 100644 --- a/parser.go +++ b/parser.go @@ -295,11 +295,11 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro } peekMatches := pythonMultiline.FindStringSubmatch(string(peekData)) - if len(peekMatches) != 3 { + // NOTE: Return if not a python-ini multi-line value. + if len(peekMatches) < 2 { return line, nil } - // NOTE: Return if not a python-ini multi-line value. currentIdentSize := len(peekMatches[1]) if currentIdentSize <= 0 { return line, nil @@ -311,7 +311,12 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro return "", err } - line += fmt.Sprintf("\n%s", peekMatches[2]) + // handle indented empty line + if len(peekMatches) < 3 { + line += "\n" + } else { + line += fmt.Sprintf("\n%s", peekMatches[2]) + } } } diff --git a/testdata/multiline.ini b/testdata/multiline.ini new file mode 100644 index 0000000..1e3255e --- /dev/null +++ b/testdata/multiline.ini @@ -0,0 +1,9 @@ +value1 = some text here + some more text here + + there is an empty line above and below + + +value2 = there is an empty line above + that is not indented so it should not be part + of the value From eeaf902516e23c6858fb369f46c342feb7e22eca Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Wed, 2 Oct 2019 17:27:33 -0700 Subject: [PATCH 02/22] added debug messages --- parser.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/parser.go b/parser.go index 9c8c70e..d5099da 100644 --- a/parser.go +++ b/parser.go @@ -297,24 +297,29 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro peekMatches := pythonMultiline.FindStringSubmatch(string(peekData)) // NOTE: Return if not a python-ini multi-line value. if len(peekMatches) < 2 { + fmt.Println("peekMatches is only ", len(peekMatches)) return line, nil } currentIdentSize := len(peekMatches[1]) if currentIdentSize <= 0 { + fmt.Println("currentIdentSize is ", currentIdentSize) return line, nil } // NOTE: Just advance the parser reader (buffer) in-sync with the peek buffer. _, err := p.readUntil('\n') if err != nil { + fmt.Println("readUntil() returned error: ", err) return "", err } // handle indented empty line if len(peekMatches) < 3 { + fmt.Println("got empty line!") line += "\n" } else { + fmt.Println("got line: ", peekMatches[2]) line += fmt.Sprintf("\n%s", peekMatches[2]) } } From 1117e812db3d978f9d8a0a14ea38450266a2d3f5 Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Wed, 2 Oct 2019 17:29:58 -0700 Subject: [PATCH 03/22] modified regex to allow empty string after indent --- parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser.go b/parser.go index d5099da..f826733 100644 --- a/parser.go +++ b/parser.go @@ -25,7 +25,7 @@ import ( "unicode" ) -var pythonMultiline = regexp.MustCompile("^(\\s+)([^\n]+)") +var pythonMultiline = regexp.MustCompile("^(\\s+)([^\n]*)") type parserOptions struct { IgnoreContinuation bool From 6621074d89b6bed3375985b0c04070949f835ff3 Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Wed, 2 Oct 2019 17:56:49 -0700 Subject: [PATCH 04/22] multiline value indentation is determined by first indented value line --- parser.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/parser.go b/parser.go index f826733..b522e93 100644 --- a/parser.go +++ b/parser.go @@ -25,7 +25,7 @@ import ( "unicode" ) -var pythonMultiline = regexp.MustCompile("^(\\s+)([^\n]*)") +var pythonMultiline = regexp.MustCompile("^(\\s)([^\n]*)") type parserOptions struct { IgnoreContinuation bool @@ -285,6 +285,7 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro parserBufferPeekResult, _ := p.buf.Peek(bufferSize) peekBuffer := bytes.NewBuffer(parserBufferPeekResult) + indentSize := 0 for { peekData, peekErr := peekBuffer.ReadBytes('\n') if peekErr != nil { @@ -301,9 +302,16 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro return line, nil } - currentIdentSize := len(peekMatches[1]) - if currentIdentSize <= 0 { - fmt.Println("currentIdentSize is ", currentIdentSize) + // determine indent size and line prefix + currentIndentSize := len(peekMatches[1]) + if indentSize < 1 { + indentSize = currentIndentSize + fmt.Println("identSize is ", indentSize) + } + + // make sure each line is indented at least as far as first line + if currentIndentSize < indentSize { + fmt.Println("currentIdentSize is ", currentIndentSize) return line, nil } @@ -315,12 +323,13 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro } // handle indented empty line + prefix := peekMatches[1][indentSize:] if len(peekMatches) < 3 { fmt.Println("got empty line!") - line += "\n" + line += "\n" + prefix } else { fmt.Println("got line: ", peekMatches[2]) - line += fmt.Sprintf("\n%s", peekMatches[2]) + line += "\n" + prefix + peekMatches[2] } } } From 933ff1d53ca6980c4a07b5b8cdad0f4887a01a6a Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Wed, 2 Oct 2019 18:43:06 -0700 Subject: [PATCH 05/22] fixed Python-style multiline value support --- parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser.go b/parser.go index b522e93..10019be 100644 --- a/parser.go +++ b/parser.go @@ -25,7 +25,7 @@ import ( "unicode" ) -var pythonMultiline = regexp.MustCompile("^(\\s)([^\n]*)") +var pythonMultiline = regexp.MustCompile("^(\\s)([^\n]+)") type parserOptions struct { IgnoreContinuation bool From 6996c91c7cb460e12557f1e227c846fe71a112a1 Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Wed, 2 Oct 2019 18:51:00 -0700 Subject: [PATCH 06/22] fixed it --- parser.go | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/parser.go b/parser.go index 10019be..e05385b 100644 --- a/parser.go +++ b/parser.go @@ -25,7 +25,11 @@ import ( "unicode" ) +<<<<<<< HEAD +var pythonMultiline = regexp.MustCompile(`^([\t\f ]+)(.*)`) +======= var pythonMultiline = regexp.MustCompile("^(\\s)([^\n]+)") +>>>>>>> 933ff1d53ca6980c4a07b5b8cdad0f4887a01a6a type parserOptions struct { IgnoreContinuation bool @@ -297,8 +301,7 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro peekMatches := pythonMultiline.FindStringSubmatch(string(peekData)) // NOTE: Return if not a python-ini multi-line value. - if len(peekMatches) < 2 { - fmt.Println("peekMatches is only ", len(peekMatches)) + if len(peekMatches) != 3 { return line, nil } @@ -306,31 +309,21 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro currentIndentSize := len(peekMatches[1]) if indentSize < 1 { indentSize = currentIndentSize - fmt.Println("identSize is ", indentSize) } // make sure each line is indented at least as far as first line if currentIndentSize < indentSize { - fmt.Println("currentIdentSize is ", currentIndentSize) return line, nil } // NOTE: Just advance the parser reader (buffer) in-sync with the peek buffer. _, err := p.readUntil('\n') if err != nil { - fmt.Println("readUntil() returned error: ", err) return "", err } // handle indented empty line - prefix := peekMatches[1][indentSize:] - if len(peekMatches) < 3 { - fmt.Println("got empty line!") - line += "\n" + prefix - } else { - fmt.Println("got line: ", peekMatches[2]) - line += "\n" + prefix + peekMatches[2] - } + line += "\n" + peekMatches[1][indentSize:] + peekMatches[2] } } From 84d07505ae60c8fa8c082cfa9ead4f81614d4887 Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Wed, 2 Oct 2019 18:53:44 -0700 Subject: [PATCH 07/22] fixed it --- parser.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/parser.go b/parser.go index e05385b..3be551f 100644 --- a/parser.go +++ b/parser.go @@ -25,11 +25,7 @@ import ( "unicode" ) -<<<<<<< HEAD var pythonMultiline = regexp.MustCompile(`^([\t\f ]+)(.*)`) -======= -var pythonMultiline = regexp.MustCompile("^(\\s)([^\n]+)") ->>>>>>> 933ff1d53ca6980c4a07b5b8cdad0f4887a01a6a type parserOptions struct { IgnoreContinuation bool From ecf5418b05750ddf8bc31be4296fe373f2efd235 Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Thu, 3 Oct 2019 16:15:35 -0700 Subject: [PATCH 08/22] cleaned up debug code --- ini.go | 6 +++++- parser.go | 57 +++++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/ini.go b/ini.go index 9de6171..0a421ea 100644 --- a/ini.go +++ b/ini.go @@ -29,7 +29,7 @@ const ( // Maximum allowed depth when recursively substituing variable names. depthValues = 99 - version = "1.48.0" + version = "1.48.0-1" ) // Version returns current package version literal. @@ -111,6 +111,10 @@ type LoadOptions struct { KeyValueDelimiters string // PreserveSurroundedQuote indicates whether to preserve surrounded quote (single and double quotes). PreserveSurroundedQuote bool + // turn on debug output (currently only useful to debug parsing Python-style multiline values) + Debug bool + // increase reader buffer size + ReaderBufferSize int } // LoadSources allows caller to apply customized options for loading from data source(s). diff --git a/parser.go b/parser.go index 3be551f..f26d7de 100644 --- a/parser.go +++ b/parser.go @@ -25,6 +25,7 @@ import ( "unicode" ) +const minReaderBufferSize = 64*1024 var pythonMultiline = regexp.MustCompile(`^([\t\f ]+)(.*)`) type parserOptions struct { @@ -35,6 +36,8 @@ type parserOptions struct { UnescapeValueDoubleQuotes bool UnescapeValueCommentSymbols bool PreserveSurroundedQuote bool + Debug bool + ReaderBufferSize int } type parser struct { @@ -47,8 +50,13 @@ type parser struct { } func newParser(r io.Reader, opts parserOptions) *parser { + size := opts.ReaderBufferSize + if size < minReaderBufferSize { + size = minReaderBufferSize + } + return &parser{ - buf: bufio.NewReader(r), + buf: bufio.NewReaderSize(r, size), options: opts, count: 1, comment: &bytes.Buffer{}, @@ -290,14 +298,37 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro peekData, peekErr := peekBuffer.ReadBytes('\n') if peekErr != nil { if peekErr == io.EOF { + if p.options.Debug { + fmt.Println("readPythonMultilines: failed to peek") + fmt.Println("readPythonMultilines: peekData is: '"+string(peekData)+"'") + fmt.Println("readPythonMultilines: value is: '"+line+"'") + } return line, nil } + if p.options.Debug { + fmt.Println("readPythonMultilines: failed to peek, returning error") + } return "", peekErr } + if p.options.Debug { + fmt.Println("readPythonMultilines: parsing '"+string(peekData)+"'") + } + peekMatches := pythonMultiline.FindStringSubmatch(string(peekData)) - // NOTE: Return if not a python-ini multi-line value. + if p.options.Debug { + fmt.Println("readPythonMultilines: matched ", len(peekMatches), " parts:") + for n, v := range peekMatches { + fmt.Println(" ", n, ": '", v, "'") + } + } + + // return if not a python-ini multi-line value. if len(peekMatches) != 3 { + if p.options.Debug { + fmt.Println("readPythonMultilines: didn't match enough parts, meaning the value has ended") + fmt.Println("readPythonMultilines: value is: '"+line+"'") + } return line, nil } @@ -305,16 +336,28 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro currentIndentSize := len(peekMatches[1]) if indentSize < 1 { indentSize = currentIndentSize + if p.options.Debug { + fmt.Println("readPythonMultilines: new indent size is ", indentSize) + } } // make sure each line is indented at least as far as first line if currentIndentSize < indentSize { + if p.options.Debug { + fmt.Println("readPythonMultilines: expected indent size is ", indentSize) + fmt.Println("readPythonMultilines: current indent size is ", currentIndentSize) + fmt.Println("readPythonMultilines: current line is not indented enough to be part of this value") + fmt.Println("readPythonMultilines: value is: '"+line+"'") + } return line, nil } - // NOTE: Just advance the parser reader (buffer) in-sync with the peek buffer. - _, err := p.readUntil('\n') + // advance the parser reader (buffer) in-sync with the peek buffer. + _, err := p.buf.Discard(len(peekData)) if err != nil { + if p.options.Debug { + fmt.Println("readPythonMultilines: failed to skip to the end, returning error") + } return "", err } @@ -333,6 +376,8 @@ func (f *File) parse(reader io.Reader) (err error) { UnescapeValueDoubleQuotes: f.options.UnescapeValueDoubleQuotes, UnescapeValueCommentSymbols: f.options.UnescapeValueCommentSymbols, PreserveSurroundedQuote: f.options.PreserveSurroundedQuote, + Debug: f.options.Debug, + ReaderBufferSize: f.options.ReaderBufferSize, }) if err = p.BOM(); err != nil { return fmt.Errorf("BOM: %v", err) @@ -356,8 +401,8 @@ func (f *File) parse(reader io.Reader) (err error) { // the size of the parser buffer is found. // TODO(unknwon): When Golang 1.10 is the lowest version supported, replace with `parserBufferSize := p.buf.Size()`. parserBufferSize := 0 - // NOTE: Peek 1kb at a time. - currentPeekSize := 1024 + // NOTE: Peek 40kb at a time. + currentPeekSize := minReaderBufferSize if f.options.AllowPythonMultilineValues { for { From 76c42888febf5bae3565cf2900515abe84bb4286 Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Thu, 3 Oct 2019 17:37:58 -0700 Subject: [PATCH 09/22] fixed imported package name for new test file --- ini_multiline_value_fix_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ini_multiline_value_fix_test.go b/ini_multiline_value_fix_test.go index 4d51e53..820a2de 100644 --- a/ini_multiline_value_fix_test.go +++ b/ini_multiline_value_fix_test.go @@ -1,7 +1,7 @@ package ini_test import ( - "github.com/r3db34n/ini" + "gopkg.in/ini.v1" . "github.com/smartystreets/goconvey/convey" "path/filepath" "testing" From e9b60432e6fba4f40361a9bd0a9cdca8ba78318f Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Thu, 3 Oct 2019 17:39:46 -0700 Subject: [PATCH 10/22] reversed change --- ini_multiline_value_fix_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ini_multiline_value_fix_test.go b/ini_multiline_value_fix_test.go index 820a2de..4d51e53 100644 --- a/ini_multiline_value_fix_test.go +++ b/ini_multiline_value_fix_test.go @@ -1,7 +1,7 @@ package ini_test import ( - "gopkg.in/ini.v1" + "github.com/r3db34n/ini" . "github.com/smartystreets/goconvey/convey" "path/filepath" "testing" From f70efdf56acbd19c1ca30269561a2b1bbab4648e Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Thu, 3 Oct 2019 17:41:43 -0700 Subject: [PATCH 11/22] fixed import --- ini_multiline_value_fix_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ini_multiline_value_fix_test.go b/ini_multiline_value_fix_test.go index 4d51e53..820a2de 100644 --- a/ini_multiline_value_fix_test.go +++ b/ini_multiline_value_fix_test.go @@ -1,7 +1,7 @@ package ini_test import ( - "github.com/r3db34n/ini" + "gopkg.in/ini.v1" . "github.com/smartystreets/goconvey/convey" "path/filepath" "testing" From 2c15745916acbbbc024d2e8251f3d2907c652190 Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Fri, 4 Oct 2019 14:07:29 -0700 Subject: [PATCH 12/22] made changes as requested by github reviewer --- ini.go | 2 +- ...ix_test.go => ini_python_multiline_test.go | 34 +++++++------------ 2 files changed, 13 insertions(+), 23 deletions(-) rename ini_multiline_value_fix_test.go => ini_python_multiline_test.go (55%) diff --git a/ini.go b/ini.go index 0a421ea..b0ca071 100644 --- a/ini.go +++ b/ini.go @@ -29,7 +29,7 @@ const ( // Maximum allowed depth when recursively substituing variable names. depthValues = 99 - version = "1.48.0-1" + version = "1.48.0" ) // Version returns current package version literal. diff --git a/ini_multiline_value_fix_test.go b/ini_python_multiline_test.go similarity index 55% rename from ini_multiline_value_fix_test.go rename to ini_python_multiline_test.go index 820a2de..81aeb87 100644 --- a/ini_multiline_value_fix_test.go +++ b/ini_python_multiline_test.go @@ -1,10 +1,11 @@ package ini_test import ( - "gopkg.in/ini.v1" - . "github.com/smartystreets/goconvey/convey" "path/filepath" "testing" + + "gopkg.in/ini.v1" + . "github.com/smartystreets/goconvey/convey" ) type testData struct { @@ -13,15 +14,6 @@ type testData struct { } func TestMultiline(t *testing.T) { - Value1 := "some text here\n"+ - "some more text here\n"+ - "\n"+ - "there is an empty line above and below\n" - - Value2 := "there is an empty line above\n"+ - "that is not indented so it should not be part\n"+ - "of the value" - Convey("Parse Python-style multiline values", t, func() { // enable Python-style multiline values @@ -31,30 +23,28 @@ func TestMultiline(t *testing.T) { // load test data path := filepath.Join("testdata", "multiline.ini") - data, e := ini.LoadSources(opts, path) + f, err := ini.LoadSources(opts, path) // Should have no error - So(e, ShouldBeNil) + So(err, ShouldBeNil) // Should have parsed data - So(data, ShouldNotBeNil) + So(f, ShouldNotBeNil) // Should have only the default section - So(len(data.Sections()), ShouldEqual, 1) + So(len(f.Sections()), ShouldEqual, 1) // Should have default section - defaultSection := data.Section("") - So(defaultSection, ShouldNotBeNil) + defaultSection := f.Section("") + So(f.Section(""), ShouldNotBeNil) // Default section should map to test data struct var testData testData e = defaultSection.MapTo(&testData) So(e, ShouldBeNil) - // 'value1' should match expected value - So(testData.Value1, ShouldEqual, Value1) - - // 'value2' should match expected value - So(testData.Value2, ShouldEqual, Value2) + // Parsed values should match expected values + So(testData.Value1, ShouldEqual, "some text here\nsome more text here\n\nthere is an empty line above and below\n") + So(testData.Value2, ShouldEqual, "there is an empty line above\nthat is not indented so it should not be part\nof the value") }) } \ No newline at end of file From a1547c40463dd03c2d162259f4d421c974c12c31 Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Fri, 4 Oct 2019 14:18:44 -0700 Subject: [PATCH 13/22] fixed build error --- ini_python_multiline_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ini_python_multiline_test.go b/ini_python_multiline_test.go index 81aeb87..29f17c7 100644 --- a/ini_python_multiline_test.go +++ b/ini_python_multiline_test.go @@ -40,8 +40,8 @@ func TestMultiline(t *testing.T) { // Default section should map to test data struct var testData testData - e = defaultSection.MapTo(&testData) - So(e, ShouldBeNil) + err = defaultSection.MapTo(&testData) + So(err, ShouldBeNil) // Parsed values should match expected values So(testData.Value1, ShouldEqual, "some text here\nsome more text here\n\nthere is an empty line above and below\n") From 3babfc9ebac5d69ab17260172578f16d69573145 Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Fri, 4 Oct 2019 16:30:15 -0700 Subject: [PATCH 14/22] added test case for large values --- ini_python_multiline_test.go | 37 +++++++++++++++++++++++------------- testdata/multiline.ini | 24 +++++++++++++++++++++++ 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/ini_python_multiline_test.go b/ini_python_multiline_test.go index 29f17c7..e603c02 100644 --- a/ini_python_multiline_test.go +++ b/ini_python_multiline_test.go @@ -11,40 +11,51 @@ import ( type testData struct { Value1 string `ini:"value1"` Value2 string `ini:"value2"` + Value3 string `ini:"value3"` } func TestMultiline(t *testing.T) { Convey("Parse Python-style multiline values", t, func() { - - // enable Python-style multiline values opts := ini.LoadOptions{ AllowPythonMultilineValues: true, } - // load test data path := filepath.Join("testdata", "multiline.ini") f, err := ini.LoadSources(opts, path) - - // Should have no error So(err, ShouldBeNil) - - // Should have parsed data So(f, ShouldNotBeNil) - - // Should have only the default section So(len(f.Sections()), ShouldEqual, 1) - // Should have default section defaultSection := f.Section("") So(f.Section(""), ShouldNotBeNil) - // Default section should map to test data struct var testData testData err = defaultSection.MapTo(&testData) So(err, ShouldBeNil) - - // Parsed values should match expected values So(testData.Value1, ShouldEqual, "some text here\nsome more text here\n\nthere is an empty line above and below\n") So(testData.Value2, ShouldEqual, "there is an empty line above\nthat is not indented so it should not be part\nof the value") + So(testData.Value3, ShouldEqual, `. + +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Eu consequat ac felis donec et odio pellentesque diam volutpat. Mauris commodo quis imperdiet massa tincidunt nunc. Interdum velit euismod in pellentesque. Nisl condimentum id venenatis a condimentum vitae sapien pellentesque. Nascetur ridiculus mus mauris vitae. Posuere urna nec tincidunt praesent semper feugiat. Lorem donec massa sapien faucibus et molestie ac feugiat sed. Ipsum dolor sit amet consectetur adipiscing elit. Enim sed faucibus turpis in eu mi. A diam sollicitudin tempor id. Quam nulla porttitor massa id neque aliquam vestibulum morbi blandit. + +Lectus sit amet est placerat in egestas. At risus viverra adipiscing at in tellus integer. Tristique senectus et netus et malesuada fames ac. In hac habitasse platea dictumst. Purus in mollis nunc sed. Pellentesque sit amet porttitor eget dolor morbi. Elit at imperdiet dui accumsan sit amet nulla. Cursus in hac habitasse platea dictumst. Bibendum arcu vitae elementum curabitur. Faucibus ornare suspendisse sed nisi lacus. In vitae turpis massa sed. Libero nunc consequat interdum varius sit amet. Molestie a iaculis at erat pellentesque. + +Dui faucibus in ornare quam viverra orci sagittis eu. Purus in mollis nunc sed id semper. Sed arcu non odio euismod lacinia at. Quis commodo odio aenean sed adipiscing diam donec. Quisque id diam vel quam elementum pulvinar. Lorem ipsum dolor sit amet. Purus ut faucibus pulvinar elementum integer enim neque volutpat ac. Fermentum posuere urna nec tincidunt praesent semper feugiat nibh sed. Gravida rutrum quisque non tellus orci. Ipsum dolor sit amet consectetur adipiscing elit pellentesque habitant. Et sollicitudin ac orci phasellus egestas tellus rutrum tellus pellentesque. Eget gravida cum sociis natoque penatibus et magnis. Elementum eu facilisis sed odio morbi quis commodo. Mollis nunc sed id semper risus in hendrerit gravida rutrum. Lorem dolor sed viverra ipsum. + +Pellentesque adipiscing commodo elit at imperdiet dui accumsan sit amet. Justo eget magna fermentum iaculis eu non diam. Condimentum mattis pellentesque id nibh tortor id aliquet lectus. Tellus molestie nunc non blandit massa enim. Mauris ultrices eros in cursus turpis. Purus viverra accumsan in nisl nisi scelerisque. Quis lectus nulla at volutpat. Purus ut faucibus pulvinar elementum integer enim. In pellentesque massa placerat duis ultricies lacus sed turpis. Elit sed vulputate mi sit amet mauris commodo. Tellus elementum sagittis vitae et. Duis tristique sollicitudin nibh sit amet commodo nulla facilisi nullam. Lectus vestibulum mattis ullamcorper velit sed ullamcorper morbi tincidunt ornare. Libero id faucibus nisl tincidunt eget nullam. Mattis aliquam faucibus purus in massa tempor. Fames ac turpis egestas sed tempus urna. Gravida in fermentum et sollicitudin ac orci phasellus egestas. + +Blandit turpis cursus in hac habitasse. Sed id semper risus in. Amet porttitor eget dolor morbi non arcu. Rhoncus est pellentesque elit ullamcorper dignissim cras tincidunt. Ut morbi tincidunt augue interdum velit. Lorem mollis aliquam ut porttitor leo a. Nunc eget lorem dolor sed viverra. Scelerisque mauris pellentesque pulvinar pellentesque. Elit at imperdiet dui accumsan sit amet. Eget magna fermentum iaculis eu non diam phasellus vestibulum lorem. Laoreet non curabitur gravida arcu ac tortor dignissim. Tortor pretium viverra suspendisse potenti nullam ac tortor vitae purus. Lacus sed viverra tellus in hac habitasse platea dictumst vestibulum. Viverra adipiscing at in tellus. Duis at tellus at urna condimentum. Eget gravida cum sociis natoque penatibus et magnis dis parturient. Pharetra massa massa ultricies mi quis hendrerit. + +Mauris pellentesque pulvinar pellentesque habitant morbi tristique. Maecenas volutpat blandit aliquam etiam. Sed turpis tincidunt id aliquet. Eget duis at tellus at urna condimentum. Pellentesque habitant morbi tristique senectus et. Amet aliquam id diam maecenas. Volutpat est velit egestas dui id. Vulputate eu scelerisque felis imperdiet proin fermentum leo vel orci. Massa sed elementum tempus egestas sed sed risus pretium. Quam quisque id diam vel quam elementum pulvinar etiam non. Sapien faucibus et molestie ac. Ipsum dolor sit amet consectetur adipiscing. Viverra orci sagittis eu volutpat. Leo urna molestie at elementum. Commodo viverra maecenas accumsan lacus. Non sodales neque sodales ut etiam sit amet. Habitant morbi tristique senectus et netus et malesuada fames. Habitant morbi tristique senectus et netus et malesuada. Blandit aliquam etiam erat velit scelerisque in. Varius duis at consectetur lorem donec massa sapien faucibus et. + +Augue mauris augue neque gravida in. Odio ut sem nulla pharetra diam sit amet nisl suscipit. Nulla aliquet enim tortor at auctor urna nunc id. Morbi tristique senectus et netus et malesuada fames ac. Quam id leo in vitae turpis massa sed elementum tempus. Ipsum faucibus vitae aliquet nec ullamcorper sit amet risus nullam. Maecenas volutpat blandit aliquam etiam erat velit scelerisque in. Sagittis nisl rhoncus mattis rhoncus urna neque viverra justo. Massa tempor nec feugiat nisl pretium. Vulputate sapien nec sagittis aliquam malesuada bibendum arcu vitae elementum. Enim lobortis scelerisque fermentum dui faucibus in ornare. Faucibus ornare suspendisse sed nisi lacus. Morbi tristique senectus et netus et malesuada fames. Malesuada pellentesque elit eget gravida cum sociis natoque penatibus et. Dictum non consectetur a erat nam at. Leo urna molestie at elementum eu facilisis sed odio morbi. Quam id leo in vitae turpis massa. Neque egestas congue quisque egestas diam in arcu. Varius morbi enim nunc faucibus a pellentesque sit. Aliquet enim tortor at auctor urna. + +Elit scelerisque mauris pellentesque pulvinar pellentesque habitant morbi tristique. Luctus accumsan tortor posuere ac. Eu ultrices vitae auctor eu augue ut lectus arcu bibendum. Pretium nibh ipsum consequat nisl vel pretium lectus. Aliquam etiam erat velit scelerisque in dictum. Sem et tortor consequat id porta nibh venenatis cras sed. A scelerisque purus semper eget duis at tellus at urna. At auctor urna nunc id. Ornare quam viverra orci sagittis eu volutpat odio. Nisl purus in mollis nunc sed id semper. Ornare suspendisse sed nisi lacus sed. Consectetur lorem donec massa sapien faucibus et. Ipsum dolor sit amet consectetur adipiscing elit ut. Porta nibh venenatis cras sed. Dignissim diam quis enim lobortis scelerisque. Quam nulla porttitor massa id. Tellus molestie nunc non blandit massa. + +Malesuada fames ac turpis egestas. Suscipit tellus mauris a diam maecenas. Turpis in eu mi bibendum neque egestas. Venenatis tellus in metus vulputate eu scelerisque felis imperdiet. Quis imperdiet massa tincidunt nunc pulvinar sapien et. Urna duis convallis convallis tellus id. Velit egestas dui id ornare arcu odio. Consectetur purus ut faucibus pulvinar elementum integer enim neque. Aenean sed adipiscing diam donec adipiscing tristique. Tortor aliquam nulla facilisi cras fermentum odio eu. Diam in arcu cursus euismod quis viverra nibh cras. + +Id ornare arcu odio ut sem. Arcu dictum varius duis at consectetur lorem donec massa sapien. Proin libero nunc consequat interdum varius sit. Ut eu sem integer vitae justo. Vitae elementum curabitur vitae nunc. Diam quam nulla porttitor massa. Lectus mauris ultrices eros in cursus turpis massa tincidunt dui. Natoque penatibus et magnis dis parturient montes. Pellentesque habitant morbi tristique senectus et netus et malesuada fames. Libero nunc consequat interdum varius sit. Rhoncus dolor purus non enim praesent. Pellentesque sit amet porttitor eget. Nibh tortor id aliquet lectus proin nibh. Fermentum iaculis eu non diam phasellus vestibulum lorem sed. + +Eu feugiat pretium nibh ipsum consequat nisl vel pretium lectus. Habitant morbi tristique senectus et netus et malesuada fames ac. Urna condimentum mattis pellentesque id. Lorem sed risus ultricies tristique nulla aliquet enim tortor at. Ipsum dolor sit amet consectetur adipiscing elit. Convallis a cras semper auctor neque vitae tempus quam. A diam sollicitudin tempor id eu nisl nunc mi ipsum. Maecenas sed enim ut sem viverra aliquet eget. Massa enim nec dui nunc mattis enim. Nam aliquam sem et tortor consequat. Adipiscing commodo elit at imperdiet dui accumsan sit amet nulla. Nullam eget felis eget nunc lobortis. Mauris a diam maecenas sed enim ut sem viverra. Ornare massa eget egestas purus. In hac habitasse platea dictumst. Ut tortor pretium viverra suspendisse potenti nullam ac tortor. Nisl nunc mi ipsum faucibus. At varius vel pharetra vel. Mauris ultrices eros in cursus turpis massa tincidunt.`) }) } \ No newline at end of file diff --git a/testdata/multiline.ini b/testdata/multiline.ini index 1e3255e..c3f6ee4 100644 --- a/testdata/multiline.ini +++ b/testdata/multiline.ini @@ -7,3 +7,27 @@ value1 = some text here value2 = there is an empty line above that is not indented so it should not be part of the value + +value3 = . + + Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Eu consequat ac felis donec et odio pellentesque diam volutpat. Mauris commodo quis imperdiet massa tincidunt nunc. Interdum velit euismod in pellentesque. Nisl condimentum id venenatis a condimentum vitae sapien pellentesque. Nascetur ridiculus mus mauris vitae. Posuere urna nec tincidunt praesent semper feugiat. Lorem donec massa sapien faucibus et molestie ac feugiat sed. Ipsum dolor sit amet consectetur adipiscing elit. Enim sed faucibus turpis in eu mi. A diam sollicitudin tempor id. Quam nulla porttitor massa id neque aliquam vestibulum morbi blandit. + + Lectus sit amet est placerat in egestas. At risus viverra adipiscing at in tellus integer. Tristique senectus et netus et malesuada fames ac. In hac habitasse platea dictumst. Purus in mollis nunc sed. Pellentesque sit amet porttitor eget dolor morbi. Elit at imperdiet dui accumsan sit amet nulla. Cursus in hac habitasse platea dictumst. Bibendum arcu vitae elementum curabitur. Faucibus ornare suspendisse sed nisi lacus. In vitae turpis massa sed. Libero nunc consequat interdum varius sit amet. Molestie a iaculis at erat pellentesque. + + Dui faucibus in ornare quam viverra orci sagittis eu. Purus in mollis nunc sed id semper. Sed arcu non odio euismod lacinia at. Quis commodo odio aenean sed adipiscing diam donec. Quisque id diam vel quam elementum pulvinar. Lorem ipsum dolor sit amet. Purus ut faucibus pulvinar elementum integer enim neque volutpat ac. Fermentum posuere urna nec tincidunt praesent semper feugiat nibh sed. Gravida rutrum quisque non tellus orci. Ipsum dolor sit amet consectetur adipiscing elit pellentesque habitant. Et sollicitudin ac orci phasellus egestas tellus rutrum tellus pellentesque. Eget gravida cum sociis natoque penatibus et magnis. Elementum eu facilisis sed odio morbi quis commodo. Mollis nunc sed id semper risus in hendrerit gravida rutrum. Lorem dolor sed viverra ipsum. + + Pellentesque adipiscing commodo elit at imperdiet dui accumsan sit amet. Justo eget magna fermentum iaculis eu non diam. Condimentum mattis pellentesque id nibh tortor id aliquet lectus. Tellus molestie nunc non blandit massa enim. Mauris ultrices eros in cursus turpis. Purus viverra accumsan in nisl nisi scelerisque. Quis lectus nulla at volutpat. Purus ut faucibus pulvinar elementum integer enim. In pellentesque massa placerat duis ultricies lacus sed turpis. Elit sed vulputate mi sit amet mauris commodo. Tellus elementum sagittis vitae et. Duis tristique sollicitudin nibh sit amet commodo nulla facilisi nullam. Lectus vestibulum mattis ullamcorper velit sed ullamcorper morbi tincidunt ornare. Libero id faucibus nisl tincidunt eget nullam. Mattis aliquam faucibus purus in massa tempor. Fames ac turpis egestas sed tempus urna. Gravida in fermentum et sollicitudin ac orci phasellus egestas. + + Blandit turpis cursus in hac habitasse. Sed id semper risus in. Amet porttitor eget dolor morbi non arcu. Rhoncus est pellentesque elit ullamcorper dignissim cras tincidunt. Ut morbi tincidunt augue interdum velit. Lorem mollis aliquam ut porttitor leo a. Nunc eget lorem dolor sed viverra. Scelerisque mauris pellentesque pulvinar pellentesque. Elit at imperdiet dui accumsan sit amet. Eget magna fermentum iaculis eu non diam phasellus vestibulum lorem. Laoreet non curabitur gravida arcu ac tortor dignissim. Tortor pretium viverra suspendisse potenti nullam ac tortor vitae purus. Lacus sed viverra tellus in hac habitasse platea dictumst vestibulum. Viverra adipiscing at in tellus. Duis at tellus at urna condimentum. Eget gravida cum sociis natoque penatibus et magnis dis parturient. Pharetra massa massa ultricies mi quis hendrerit. + + Mauris pellentesque pulvinar pellentesque habitant morbi tristique. Maecenas volutpat blandit aliquam etiam. Sed turpis tincidunt id aliquet. Eget duis at tellus at urna condimentum. Pellentesque habitant morbi tristique senectus et. Amet aliquam id diam maecenas. Volutpat est velit egestas dui id. Vulputate eu scelerisque felis imperdiet proin fermentum leo vel orci. Massa sed elementum tempus egestas sed sed risus pretium. Quam quisque id diam vel quam elementum pulvinar etiam non. Sapien faucibus et molestie ac. Ipsum dolor sit amet consectetur adipiscing. Viverra orci sagittis eu volutpat. Leo urna molestie at elementum. Commodo viverra maecenas accumsan lacus. Non sodales neque sodales ut etiam sit amet. Habitant morbi tristique senectus et netus et malesuada fames. Habitant morbi tristique senectus et netus et malesuada. Blandit aliquam etiam erat velit scelerisque in. Varius duis at consectetur lorem donec massa sapien faucibus et. + + Augue mauris augue neque gravida in. Odio ut sem nulla pharetra diam sit amet nisl suscipit. Nulla aliquet enim tortor at auctor urna nunc id. Morbi tristique senectus et netus et malesuada fames ac. Quam id leo in vitae turpis massa sed elementum tempus. Ipsum faucibus vitae aliquet nec ullamcorper sit amet risus nullam. Maecenas volutpat blandit aliquam etiam erat velit scelerisque in. Sagittis nisl rhoncus mattis rhoncus urna neque viverra justo. Massa tempor nec feugiat nisl pretium. Vulputate sapien nec sagittis aliquam malesuada bibendum arcu vitae elementum. Enim lobortis scelerisque fermentum dui faucibus in ornare. Faucibus ornare suspendisse sed nisi lacus. Morbi tristique senectus et netus et malesuada fames. Malesuada pellentesque elit eget gravida cum sociis natoque penatibus et. Dictum non consectetur a erat nam at. Leo urna molestie at elementum eu facilisis sed odio morbi. Quam id leo in vitae turpis massa. Neque egestas congue quisque egestas diam in arcu. Varius morbi enim nunc faucibus a pellentesque sit. Aliquet enim tortor at auctor urna. + + Elit scelerisque mauris pellentesque pulvinar pellentesque habitant morbi tristique. Luctus accumsan tortor posuere ac. Eu ultrices vitae auctor eu augue ut lectus arcu bibendum. Pretium nibh ipsum consequat nisl vel pretium lectus. Aliquam etiam erat velit scelerisque in dictum. Sem et tortor consequat id porta nibh venenatis cras sed. A scelerisque purus semper eget duis at tellus at urna. At auctor urna nunc id. Ornare quam viverra orci sagittis eu volutpat odio. Nisl purus in mollis nunc sed id semper. Ornare suspendisse sed nisi lacus sed. Consectetur lorem donec massa sapien faucibus et. Ipsum dolor sit amet consectetur adipiscing elit ut. Porta nibh venenatis cras sed. Dignissim diam quis enim lobortis scelerisque. Quam nulla porttitor massa id. Tellus molestie nunc non blandit massa. + + Malesuada fames ac turpis egestas. Suscipit tellus mauris a diam maecenas. Turpis in eu mi bibendum neque egestas. Venenatis tellus in metus vulputate eu scelerisque felis imperdiet. Quis imperdiet massa tincidunt nunc pulvinar sapien et. Urna duis convallis convallis tellus id. Velit egestas dui id ornare arcu odio. Consectetur purus ut faucibus pulvinar elementum integer enim neque. Aenean sed adipiscing diam donec adipiscing tristique. Tortor aliquam nulla facilisi cras fermentum odio eu. Diam in arcu cursus euismod quis viverra nibh cras. + + Id ornare arcu odio ut sem. Arcu dictum varius duis at consectetur lorem donec massa sapien. Proin libero nunc consequat interdum varius sit. Ut eu sem integer vitae justo. Vitae elementum curabitur vitae nunc. Diam quam nulla porttitor massa. Lectus mauris ultrices eros in cursus turpis massa tincidunt dui. Natoque penatibus et magnis dis parturient montes. Pellentesque habitant morbi tristique senectus et netus et malesuada fames. Libero nunc consequat interdum varius sit. Rhoncus dolor purus non enim praesent. Pellentesque sit amet porttitor eget. Nibh tortor id aliquet lectus proin nibh. Fermentum iaculis eu non diam phasellus vestibulum lorem sed. + + Eu feugiat pretium nibh ipsum consequat nisl vel pretium lectus. Habitant morbi tristique senectus et netus et malesuada fames ac. Urna condimentum mattis pellentesque id. Lorem sed risus ultricies tristique nulla aliquet enim tortor at. Ipsum dolor sit amet consectetur adipiscing elit. Convallis a cras semper auctor neque vitae tempus quam. A diam sollicitudin tempor id eu nisl nunc mi ipsum. Maecenas sed enim ut sem viverra aliquet eget. Massa enim nec dui nunc mattis enim. Nam aliquam sem et tortor consequat. Adipiscing commodo elit at imperdiet dui accumsan sit amet nulla. Nullam eget felis eget nunc lobortis. Mauris a diam maecenas sed enim ut sem viverra. Ornare massa eget egestas purus. In hac habitasse platea dictumst. Ut tortor pretium viverra suspendisse potenti nullam ac tortor. Nisl nunc mi ipsum faucibus. At varius vel pharetra vel. Mauris ultrices eros in cursus turpis massa tincidunt. From dba750a4e43ab6ae73cdda763a70ecd7d757ae70 Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Fri, 4 Oct 2019 16:58:35 -0700 Subject: [PATCH 15/22] incorporated second round of comments --- ini_python_multiline_test.go | 9 +++------ parser.go | 16 ++++++++-------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/ini_python_multiline_test.go b/ini_python_multiline_test.go index e603c02..dc3c7c0 100644 --- a/ini_python_multiline_test.go +++ b/ini_python_multiline_test.go @@ -4,7 +4,8 @@ import ( "path/filepath" "testing" - "gopkg.in/ini.v1" + "github.com/r3db34n/ini" +// "gopkg.in/ini.v1" . "github.com/smartystreets/goconvey/convey" ) @@ -16,12 +17,8 @@ type testData struct { func TestMultiline(t *testing.T) { Convey("Parse Python-style multiline values", t, func() { - opts := ini.LoadOptions{ - AllowPythonMultilineValues: true, - } - path := filepath.Join("testdata", "multiline.ini") - f, err := ini.LoadSources(opts, path) + f, err := ini.LoadSources(ini.LoadOptions{AllowPythonMultilineValues: true, ReaderBufferSize: 64*1024}, path) So(err, ShouldBeNil) So(f, ShouldNotBeNil) So(len(f.Sections()), ShouldEqual, 1) diff --git a/parser.go b/parser.go index f26d7de..4334daf 100644 --- a/parser.go +++ b/parser.go @@ -25,7 +25,7 @@ import ( "unicode" ) -const minReaderBufferSize = 64*1024 +const minReaderBufferSize = 4096 var pythonMultiline = regexp.MustCompile(`^([\t\f ]+)(.*)`) type parserOptions struct { @@ -323,7 +323,7 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro } } - // return if not a python-ini multi-line value. + // Return if not a Python multiline value. if len(peekMatches) != 3 { if p.options.Debug { fmt.Println("readPythonMultilines: didn't match enough parts, meaning the value has ended") @@ -332,7 +332,7 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro return line, nil } - // determine indent size and line prefix + // Determine indent size and line prefix. currentIndentSize := len(peekMatches[1]) if indentSize < 1 { indentSize = currentIndentSize @@ -341,7 +341,7 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro } } - // make sure each line is indented at least as far as first line + // Make sure each line is indented at least as far as first line. if currentIndentSize < indentSize { if p.options.Debug { fmt.Println("readPythonMultilines: expected indent size is ", indentSize) @@ -352,7 +352,7 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro return line, nil } - // advance the parser reader (buffer) in-sync with the peek buffer. + // Advance the parser reader (buffer) in-sync with the peek buffer. _, err := p.buf.Discard(len(peekData)) if err != nil { if p.options.Debug { @@ -361,7 +361,7 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro return "", err } - // handle indented empty line + // Handle indented empty line. line += "\n" + peekMatches[1][indentSize:] + peekMatches[2] } } @@ -399,9 +399,9 @@ func (f *File) parse(reader io.Reader) (err error) { // NOTE: Iterate and increase `currentPeekSize` until // the size of the parser buffer is found. - // TODO(unknwon): When Golang 1.10 is the lowest version supported, replace with `parserBufferSize := p.buf.Size()`. + // TODO(unknown): When Golang 1.10 is the lowest version supported, replace with `parserBufferSize := p.buf.Size()`. parserBufferSize := 0 - // NOTE: Peek 40kb at a time. + // NOTE: Peek 4kb at a time. currentPeekSize := minReaderBufferSize if f.options.AllowPythonMultilineValues { From e8fffdded4534e12540f9f22dbaa209bf1bcf207 Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Fri, 4 Oct 2019 17:00:18 -0700 Subject: [PATCH 16/22] fixed import to point to original repo --- ini_python_multiline_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ini_python_multiline_test.go b/ini_python_multiline_test.go index dc3c7c0..20a512b 100644 --- a/ini_python_multiline_test.go +++ b/ini_python_multiline_test.go @@ -4,8 +4,7 @@ import ( "path/filepath" "testing" - "github.com/r3db34n/ini" -// "gopkg.in/ini.v1" + "gopkg.in/ini.v1" . "github.com/smartystreets/goconvey/convey" ) From 09a95b4d3061e9f9cdd3ca6c699290ba0344f318 Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Sat, 5 Oct 2019 23:24:33 -0700 Subject: [PATCH 17/22] made Debug option a function to allow custom implementation made other changes as requested --- ini.go | 9 ++++--- ini_python_multiline_test.go | 6 ++++- parser.go | 50 ++++++++++++++++++------------------ 3 files changed, 36 insertions(+), 29 deletions(-) diff --git a/ini.go b/ini.go index b0ca071..ce155c2 100644 --- a/ini.go +++ b/ini.go @@ -111,12 +111,15 @@ type LoadOptions struct { KeyValueDelimiters string // PreserveSurroundedQuote indicates whether to preserve surrounded quote (single and double quotes). PreserveSurroundedQuote bool - // turn on debug output (currently only useful to debug parsing Python-style multiline values) - Debug bool - // increase reader buffer size + // Debug indicates whether to collect debug information (currently only useful to debug parsing Python-style multiline values). + Debug DebugFunc + // ReaderBufferSize is the buffer size of the reader in bytes. ReaderBufferSize int } +// DebugFunc is the type of function called to log parse events +type DebugFunc func(args ...interface{}) + // LoadSources allows caller to apply customized options for loading from data source(s). func LoadSources(opts LoadOptions, source interface{}, others ...interface{}) (_ *File, err error) { sources := make([]dataSource, len(others)+1) diff --git a/ini_python_multiline_test.go b/ini_python_multiline_test.go index 20a512b..a39d8e0 100644 --- a/ini_python_multiline_test.go +++ b/ini_python_multiline_test.go @@ -1,6 +1,7 @@ package ini_test import ( + "github.com/r3db34n/ini.tmp" "path/filepath" "testing" @@ -17,7 +18,10 @@ type testData struct { func TestMultiline(t *testing.T) { Convey("Parse Python-style multiline values", t, func() { path := filepath.Join("testdata", "multiline.ini") - f, err := ini.LoadSources(ini.LoadOptions{AllowPythonMultilineValues: true, ReaderBufferSize: 64*1024}, path) + f, err := ini.LoadSources(ini.LoadOptions{ + AllowPythonMultilineValues: true, + ReaderBufferSize: 64*1024, + }, path) So(err, ShouldBeNil) So(f, ShouldNotBeNil) So(len(f.Sections()), ShouldEqual, 1) diff --git a/parser.go b/parser.go index 4334daf..1eb7ae7 100644 --- a/parser.go +++ b/parser.go @@ -36,7 +36,7 @@ type parserOptions struct { UnescapeValueDoubleQuotes bool UnescapeValueCommentSymbols bool PreserveSurroundedQuote bool - Debug bool + Debug DebugFunc ReaderBufferSize int } @@ -298,36 +298,36 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro peekData, peekErr := peekBuffer.ReadBytes('\n') if peekErr != nil { if peekErr == io.EOF { - if p.options.Debug { - fmt.Println("readPythonMultilines: failed to peek") - fmt.Println("readPythonMultilines: peekData is: '"+string(peekData)+"'") - fmt.Println("readPythonMultilines: value is: '"+line+"'") + if p.options.Debug != nil { + p.options.Debug("readPythonMultilines: failed to peek") + p.options.Debug("readPythonMultilines: peekData is: '"+string(peekData)+"'") + p.options.Debug("readPythonMultilines: value is: '"+line+"'") } return line, nil } - if p.options.Debug { - fmt.Println("readPythonMultilines: failed to peek, returning error") + if p.options.Debug != nil { + p.options.Debug("readPythonMultilines: failed to peek, returning error") } return "", peekErr } - if p.options.Debug { - fmt.Println("readPythonMultilines: parsing '"+string(peekData)+"'") + if p.options.Debug != nil { + p.options.Debug("readPythonMultilines: parsing '"+string(peekData)+"'") } peekMatches := pythonMultiline.FindStringSubmatch(string(peekData)) - if p.options.Debug { - fmt.Println("readPythonMultilines: matched ", len(peekMatches), " parts:") + if p.options.Debug != nil { + p.options.Debug("readPythonMultilines: matched ", len(peekMatches), " parts:") for n, v := range peekMatches { - fmt.Println(" ", n, ": '", v, "'") + p.options.Debug(" ", n, ": '", v, "'") } } // Return if not a Python multiline value. if len(peekMatches) != 3 { - if p.options.Debug { - fmt.Println("readPythonMultilines: didn't match enough parts, meaning the value has ended") - fmt.Println("readPythonMultilines: value is: '"+line+"'") + if p.options.Debug != nil { + p.options.Debug("readPythonMultilines: didn't match enough parts, meaning the value has ended") + p.options.Debug("readPythonMultilines: value is: '"+line+"'") } return line, nil } @@ -336,18 +336,18 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro currentIndentSize := len(peekMatches[1]) if indentSize < 1 { indentSize = currentIndentSize - if p.options.Debug { - fmt.Println("readPythonMultilines: new indent size is ", indentSize) + if p.options.Debug != nil { + p.options.Debug("readPythonMultilines: new indent size is ", indentSize) } } // Make sure each line is indented at least as far as first line. if currentIndentSize < indentSize { - if p.options.Debug { - fmt.Println("readPythonMultilines: expected indent size is ", indentSize) - fmt.Println("readPythonMultilines: current indent size is ", currentIndentSize) - fmt.Println("readPythonMultilines: current line is not indented enough to be part of this value") - fmt.Println("readPythonMultilines: value is: '"+line+"'") + if p.options.Debug != nil { + p.options.Debug("readPythonMultilines: expected indent size is ", indentSize) + p.options.Debug("readPythonMultilines: current indent size is ", currentIndentSize) + p.options.Debug("readPythonMultilines: current line is not indented enough to be part of this value") + p.options.Debug("readPythonMultilines: value is: '"+line+"'") } return line, nil } @@ -355,8 +355,8 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro // Advance the parser reader (buffer) in-sync with the peek buffer. _, err := p.buf.Discard(len(peekData)) if err != nil { - if p.options.Debug { - fmt.Println("readPythonMultilines: failed to skip to the end, returning error") + if p.options.Debug != nil { + p.options.Debug("readPythonMultilines: failed to skip to the end, returning error") } return "", err } @@ -399,7 +399,7 @@ func (f *File) parse(reader io.Reader) (err error) { // NOTE: Iterate and increase `currentPeekSize` until // the size of the parser buffer is found. - // TODO(unknown): When Golang 1.10 is the lowest version supported, replace with `parserBufferSize := p.buf.Size()`. + // TODO(unknwon): When Golang 1.10 is the lowest version supported, replace with `parserBufferSize := p.buf.Size()`. parserBufferSize := 0 // NOTE: Peek 4kb at a time. currentPeekSize := minReaderBufferSize From e1b731c5b2a06284fce555b83ce9e86e151dfc33 Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Sat, 5 Oct 2019 23:31:25 -0700 Subject: [PATCH 18/22] removed bogus import (added by GoLand) --- ini_python_multiline_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/ini_python_multiline_test.go b/ini_python_multiline_test.go index a39d8e0..45fe34a 100644 --- a/ini_python_multiline_test.go +++ b/ini_python_multiline_test.go @@ -1,7 +1,6 @@ package ini_test import ( - "github.com/r3db34n/ini.tmp" "path/filepath" "testing" From b538da98c2cf8196b9707d522f7fdd01f5a3bd47 Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Sun, 6 Oct 2019 18:34:31 -0700 Subject: [PATCH 19/22] simplified debug function args --- ini.go | 4 +-- ini_python_multiline_test.go | 5 ++++ parser.go | 54 +++++++++++++++--------------------- 3 files changed, 29 insertions(+), 34 deletions(-) diff --git a/ini.go b/ini.go index ce155c2..00f3801 100644 --- a/ini.go +++ b/ini.go @@ -117,8 +117,8 @@ type LoadOptions struct { ReaderBufferSize int } -// DebugFunc is the type of function called to log parse events -type DebugFunc func(args ...interface{}) +// DebugFunc is the type of function called to log parse events. +type DebugFunc func(message string) // LoadSources allows caller to apply customized options for loading from data source(s). func LoadSources(opts LoadOptions, source interface{}, others ...interface{}) (_ *File, err error) { diff --git a/ini_python_multiline_test.go b/ini_python_multiline_test.go index 45fe34a..da925c4 100644 --- a/ini_python_multiline_test.go +++ b/ini_python_multiline_test.go @@ -20,6 +20,11 @@ func TestMultiline(t *testing.T) { f, err := ini.LoadSources(ini.LoadOptions{ AllowPythonMultilineValues: true, ReaderBufferSize: 64*1024, + /* + Debug: func(m string) { + fmt.Println(m) + }, + */ }, path) So(err, ShouldBeNil) So(f, ShouldNotBeNil) diff --git a/parser.go b/parser.go index 1eb7ae7..250e352 100644 --- a/parser.go +++ b/parser.go @@ -49,6 +49,12 @@ type parser struct { comment *bytes.Buffer } +func (p *parser) debug(args ...interface{}) { + if p.options.Debug != nil { + p.options.Debug(fmt.Sprint(args...)) + } +} + func newParser(r io.Reader, opts parserOptions) *parser { size := opts.ReaderBufferSize if size < minReaderBufferSize { @@ -298,37 +304,27 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro peekData, peekErr := peekBuffer.ReadBytes('\n') if peekErr != nil { if peekErr == io.EOF { - if p.options.Debug != nil { - p.options.Debug("readPythonMultilines: failed to peek") - p.options.Debug("readPythonMultilines: peekData is: '"+string(peekData)+"'") - p.options.Debug("readPythonMultilines: value is: '"+line+"'") - } + p.debug("readPythonMultilines: failed to peek") + p.debug("readPythonMultilines: peekData is: '"+string(peekData)+"'") + p.debug("readPythonMultilines: value is: '"+line+"'") return line, nil } - if p.options.Debug != nil { - p.options.Debug("readPythonMultilines: failed to peek, returning error") - } + p.debug("readPythonMultilines: failed to peek, returning error") return "", peekErr } - if p.options.Debug != nil { - p.options.Debug("readPythonMultilines: parsing '"+string(peekData)+"'") - } + p.debug("readPythonMultilines: parsing '"+string(peekData)+"'") peekMatches := pythonMultiline.FindStringSubmatch(string(peekData)) - if p.options.Debug != nil { - p.options.Debug("readPythonMultilines: matched ", len(peekMatches), " parts:") - for n, v := range peekMatches { - p.options.Debug(" ", n, ": '", v, "'") - } + p.debug("readPythonMultilines: matched ", len(peekMatches), " parts:") + for n, v := range peekMatches { + p.debug(" ", n, ": '", v, "'") } // Return if not a Python multiline value. if len(peekMatches) != 3 { - if p.options.Debug != nil { - p.options.Debug("readPythonMultilines: didn't match enough parts, meaning the value has ended") - p.options.Debug("readPythonMultilines: value is: '"+line+"'") - } + p.debug("readPythonMultilines: didn't match enough parts, meaning the value has ended") + p.debug("readPythonMultilines: value is: '"+line+"'") return line, nil } @@ -336,28 +332,22 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro currentIndentSize := len(peekMatches[1]) if indentSize < 1 { indentSize = currentIndentSize - if p.options.Debug != nil { - p.options.Debug("readPythonMultilines: new indent size is ", indentSize) - } + p.debug("readPythonMultilines: new indent size is ", indentSize) } // Make sure each line is indented at least as far as first line. if currentIndentSize < indentSize { - if p.options.Debug != nil { - p.options.Debug("readPythonMultilines: expected indent size is ", indentSize) - p.options.Debug("readPythonMultilines: current indent size is ", currentIndentSize) - p.options.Debug("readPythonMultilines: current line is not indented enough to be part of this value") - p.options.Debug("readPythonMultilines: value is: '"+line+"'") - } + p.debug("readPythonMultilines: expected indent size is ", indentSize) + p.debug("readPythonMultilines: current indent size is ", currentIndentSize) + p.debug("readPythonMultilines: current line is not indented enough to be part of this value") + p.debug("readPythonMultilines: value is: '"+line+"'") return line, nil } // Advance the parser reader (buffer) in-sync with the peek buffer. _, err := p.buf.Discard(len(peekData)) if err != nil { - if p.options.Debug != nil { - p.options.Debug("readPythonMultilines: failed to skip to the end, returning error") - } + p.debug("readPythonMultilines: failed to skip to the end, returning error") return "", err } From 6833dce49da4d81e62a1a958b6269119d57aa6da Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Sat, 26 Oct 2019 17:43:40 -0700 Subject: [PATCH 20/22] made changes as requested --- ini.go | 4 ++-- parser.go | 29 ++++++++++++----------------- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/ini.go b/ini.go index 00f3801..0d84ffa 100644 --- a/ini.go +++ b/ini.go @@ -111,8 +111,8 @@ type LoadOptions struct { KeyValueDelimiters string // PreserveSurroundedQuote indicates whether to preserve surrounded quote (single and double quotes). PreserveSurroundedQuote bool - // Debug indicates whether to collect debug information (currently only useful to debug parsing Python-style multiline values). - Debug DebugFunc + // DebugFunc is called to collect debug information (currently only useful to debug parsing Python-style multiline values). + DebugFunc DebugFunc // ReaderBufferSize is the buffer size of the reader in bytes. ReaderBufferSize int } diff --git a/parser.go b/parser.go index 250e352..f93304c 100644 --- a/parser.go +++ b/parser.go @@ -36,7 +36,7 @@ type parserOptions struct { UnescapeValueDoubleQuotes bool UnescapeValueCommentSymbols bool PreserveSurroundedQuote bool - Debug DebugFunc + DebugFunc DebugFunc ReaderBufferSize int } @@ -49,9 +49,9 @@ type parser struct { comment *bytes.Buffer } -func (p *parser) debug(args ...interface{}) { - if p.options.Debug != nil { - p.options.Debug(fmt.Sprint(args...)) +func (p *parser) debug(format string, args ...interface{}) { + if p.options.DebugFunc != nil { + p.options.DebugFunc(fmt.Sprintf(format, args...)) } } @@ -304,27 +304,25 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro peekData, peekErr := peekBuffer.ReadBytes('\n') if peekErr != nil { if peekErr == io.EOF { - p.debug("readPythonMultilines: failed to peek") - p.debug("readPythonMultilines: peekData is: '"+string(peekData)+"'") - p.debug("readPythonMultilines: value is: '"+line+"'") + p.debug("readPythonMultilines: io.EOF, peekData: %q, line: %q", string(peekData), line) return line, nil } + p.debug("readPythonMultilines: failed to peek, returning error") return "", peekErr } - p.debug("readPythonMultilines: parsing '"+string(peekData)+"'") + p.debug("readPythonMultilines: parsing %q", string(peekData)) peekMatches := pythonMultiline.FindStringSubmatch(string(peekData)) - p.debug("readPythonMultilines: matched ", len(peekMatches), " parts:") + p.debug("readPythonMultilines: matched %d parts", len(peekMatches)) for n, v := range peekMatches { p.debug(" ", n, ": '", v, "'") } // Return if not a Python multiline value. if len(peekMatches) != 3 { - p.debug("readPythonMultilines: didn't match enough parts, meaning the value has ended") - p.debug("readPythonMultilines: value is: '"+line+"'") + p.debug("readPythonMultilines: end of value, got: %q", line) return line, nil } @@ -332,15 +330,12 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro currentIndentSize := len(peekMatches[1]) if indentSize < 1 { indentSize = currentIndentSize - p.debug("readPythonMultilines: new indent size is ", indentSize) + p.debug("readPythonMultilines: indent size is %d", indentSize) } // Make sure each line is indented at least as far as first line. if currentIndentSize < indentSize { - p.debug("readPythonMultilines: expected indent size is ", indentSize) - p.debug("readPythonMultilines: current indent size is ", currentIndentSize) - p.debug("readPythonMultilines: current line is not indented enough to be part of this value") - p.debug("readPythonMultilines: value is: '"+line+"'") + p.debug("readPythonMultilines: end of value, current indent: %d, expected indent: %d, line: %q", currentIndentSize, indentSize, line) return line, nil } @@ -366,7 +361,7 @@ func (f *File) parse(reader io.Reader) (err error) { UnescapeValueDoubleQuotes: f.options.UnescapeValueDoubleQuotes, UnescapeValueCommentSymbols: f.options.UnescapeValueCommentSymbols, PreserveSurroundedQuote: f.options.PreserveSurroundedQuote, - Debug: f.options.Debug, + DebugFunc: f.options.DebugFunc, ReaderBufferSize: f.options.ReaderBufferSize, }) if err = p.BOM(); err != nil { From f5487cb00cae60d49cbd8ea804a0ae5c6d90b9cc Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Sat, 26 Oct 2019 17:52:38 -0700 Subject: [PATCH 21/22] fixed format error --- parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser.go b/parser.go index f93304c..0499987 100644 --- a/parser.go +++ b/parser.go @@ -317,7 +317,7 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro peekMatches := pythonMultiline.FindStringSubmatch(string(peekData)) p.debug("readPythonMultilines: matched %d parts", len(peekMatches)) for n, v := range peekMatches { - p.debug(" ", n, ": '", v, "'") + p.debug(" %d: %q", n, v) } // Return if not a Python multiline value. From 505ffc63fecde5005ff269e211212a4e6a835462 Mon Sep 17 00:00:00 2001 From: Joerg Reichelt Date: Tue, 5 Nov 2019 15:52:12 -0800 Subject: [PATCH 22/22] made requested changes --- parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser.go b/parser.go index 0499987..9995d31 100644 --- a/parser.go +++ b/parser.go @@ -308,7 +308,7 @@ func (p *parser) readPythonMultilines(line string, bufferSize int) (string, erro return line, nil } - p.debug("readPythonMultilines: failed to peek, returning error") + p.debug("readPythonMultilines: failed to peek with error: %v", peekErr) return "", peekErr }