使用 Golang 从文本文件末尾读取特定内容的有效方法

Question

我有一个很大的文本日志文件，其中包含由一些特殊字符分隔的两部分，就像

...
this is the very large 
part, contains lots lines.

#SPECIAL CHARS START#

...
this is the small part the the end, 
contain several lines, but 
we do not know how many lines
this part contains

我的要求是获取#SPECIAL CHARS START#之后一直到最后的小部分文本内容，如何用Golang有效获取？

更新： 我目前的解决方案是从文件末尾逐行获取并记住光标，如果该行包含特殊字符，则打破循环并获取光标

func getBackwardLine(file *os.File, start int64) (string, int64) {
    line := ""
    cursor :=start
    stat, _ := file.Stat()
    filesize := stat.Size()

    for { 
        cursor--
        file.Seek(cursor, io.SeekEnd)

        char := make([]byte, 1)
        file.Read(char)

        if cursor != -1 && (char[0] == 10 || char[0] == 13) { 
            break
        }

        line = fmt.Sprintf("%s%s", string(char), line) 

        if cursor == -filesize { 
            break
        }
    }
    return line, cursor

}

func main() {
    file, err := os.Open("some.log")
    if err != nil {
        os.Exit(1)
    }
    defer file.Close()

    var cursor int64 = 0
    var line = ""

    for {  
        line, cursor = getBackwardLine(file, cursor)
        fmt.Println(line)
        if(strings.Contains(line, "#SPECIAL CHARS START#")) {
            break
        }
    }


    fmt.Println(cursor)  //now we get the cursor for the start of special characters
}

Answer 1

此解决方案实现了向后阅读器。

它以b.Len字节为单位从末尾开始读取文件，然后在块内寻找分隔符，当前为\n，然后将起始偏移量提前SepIndex（这是为了防止将搜索字符串拆分为两次连续读取）。在继续读取下一个块之前，它会在块读取中查找search 字符串，如果找到，它会返回其在文件中的起始位置并停止。 否则，它将起始偏移量减少b.Len，然后读取下一个块。

只要您的搜索字符串位于文件的最后 40%，您应该会获得更好的性能，但这是经过实战考验的。

如果您的搜索字符串在最后 10% 以内，我相信您会获胜。

main.go

package main

import (
    "bytes"
    "flag"
    "fmt"
    "io"
    "log"
    "os"
    "time"

    "github.com/mattetti/filebuffer"
)

func main() {

    var search string
    var sep string
    var verbose bool
    flag.StringVar(&search, "search", "findme", "search word")
    flag.StringVar(&sep, "sep", "\n", "separator for the search detection")
    flag.BoolVar(&verbose, "v", false, "verbosity")
    flag.Parse()

    d := make(chan struct{})
    b := &bytes.Buffer{}
    go func() {
        io.Copy(b, os.Stdin)
        d <- struct{}{}
    }()
    <-time.After(time.Millisecond)
    select {
    case <-d:
    default:
        os.Stdin.Close()
    }

    readSize := 1024
    if b.Len() < 1 {
        input := fmt.Sprintf("%s%stail content", bytes.Repeat([]byte(" "), readSize-5), search)
        input += input
        b.WriteString(input)
    }

    bsearch := []byte(search)
    s, err := bytesSearch(b.Bytes())
    if err != nil {
        log.Fatal(err)
    }
    if verbose {
        s.logger = log.New(os.Stderr, "", log.LstdFlags)
    }
    s.Buffer = make([]byte, readSize)
    s.Sep = []byte(sep)
    got, err := s.Index(bsearch)
    if err != nil {
        log.Fatal(err)
    }
    fmt.Println("Index ", got)
    got, err = s.Index2(bsearch)
    if err != nil {
        log.Fatal(err)
    }
    fmt.Println("Index ", got)

}

type tailSearch struct {
    F      io.ReadSeeker
    Buffer []byte
    Sep    []byte
    start  int64
    logger interface {
        Println(...interface{})
    }
}

func fileSearch(f *os.File) (ret tailSearch, err error) {
    ret.F = f
    st, err := f.Stat()
    if err != nil {
        return
    }
    ret.start = st.Size()
    ret.Sep = []byte("\n")
    return ret, nil
}

func bytesSearch(b []byte) (ret tailSearch, err error) {
    ret.F = filebuffer.New(b)
    ret.start = int64(len(b))
    ret.Sep = []byte("\n")
    return
}

func (b tailSearch) Index(search []byte) (int64, error) {

    if b.Buffer == nil {
        b.Buffer = make([]byte, 1024, 1024)
    }
    buf := b.Buffer
    blen := len(b.Buffer)

    hasended := false
    for !hasended {
        if b.logger != nil {
            b.logger.Println("a start", b.start)
        }
        offset := b.start - int64(blen)
        if offset < 0 {
            offset = 0
            hasended = true
        }
        _, err := b.F.Seek(offset, 0)
        if err != nil {
            hasended = true
        }
        n, err := b.F.Read(buf)
        if b.logger != nil {
            b.logger.Println("f n", n, "err", err)
        }
        if err != nil {
            hasended = true
        }
        buf = buf[:n]
        b.start -= int64(n)
        if b.logger != nil {
            b.logger.Println("g start", b.start)
        }
        if b.start > 0 {
            i := bytes.Index(buf, b.Sep)
            if b.logger != nil {
                b.logger.Println("h sep", i)
            }
            if i > -1 {
                b.start += int64(i)
                buf = buf[i:]
                if b.logger != nil {
                    b.logger.Println("i start", b.start)
                }
            }
        }
        if e := bytes.LastIndex(buf, search); e > -1 {
            return b.start + int64(e), nil
        }
    }

    return -1, nil
}

func (b tailSearch) Index2(search []byte) (int64, error) {

    if b.Buffer == nil {
        b.Buffer = make([]byte, 1024, 1024)
    }
    buf := b.Buffer
    blen := len(b.Buffer)

    hasended := false
    for !hasended {
        if b.logger != nil {
            b.logger.Println("a start", b.start)
        }
        offset := b.start - int64(blen)
        if offset < 0 {
            offset = 0
            hasended = true
        }
        _, err := b.F.Seek(offset, 0)
        if err != nil {
            hasended = true
        }

        n, err := b.F.Read(buf)
        if b.logger != nil {
            b.logger.Println("f n", n, "err", err)
        }
        if err != nil {
            hasended = true
        }
        buf = buf[:n]
        b.start -= int64(n)

        if b.logger != nil {
            b.logger.Println("g start", b.start)
        }

        for i := 1; i < len(search); i++ {
            if bytes.HasPrefix(buf, search[i:]) {
                e := i - len(search)
                b.start += int64(e)
                buf = buf[e:]
            }
        }
        if b.logger != nil {
            b.logger.Println("g start", b.start)
        }

        if e := bytes.LastIndex(buf, search); e > -1 {
            return b.start + int64(e), nil
        }
    }

    return -1, nil
}

main_test.go

package main

import (
    "bytes"
    "fmt"
    "strings"
    "testing"
)

func TestOne(t *testing.T) {

    type test struct {
        search  []byte
        readLen int
        input   string
        sep     []byte
        want    int64
    }

    search := []byte("find me")
    blockLen := 1024
    tests := []test{
        test{
            search:  search,
            sep:     []byte("\n"),
            readLen: blockLen,
            input:   fmt.Sprintf("%stail content", search),
            want:    0,
        },
        test{
            search:  search,
            sep:     []byte("\n"),
            readLen: blockLen,
            input:   fmt.Sprintf(""),
            want:    -1,
        },
        test{
            search:  search,
            sep:     []byte("\n"),
            readLen: blockLen,
            input:   strings.Repeat("nop\n", 10000),
            want:    -1,
        },
        test{
            search:  search,
            sep:     []byte("\n"),
            readLen: blockLen,
            input:   fmt.Sprintf("%s%stail content", bytes.Repeat([]byte(" "), blockLen-5), search),
            want:    1019,
        },
        test{
            search:  search,
            sep:     []byte("\n"),
            readLen: blockLen,
            input:   fmt.Sprintf("%s%stail content", bytes.Repeat([]byte(" "), blockLen), search),
            want:    1024,
        },
        test{
            search:  search,
            sep:     []byte("\n"),
            readLen: blockLen,
            input:   fmt.Sprintf("%s%stail content", bytes.Repeat([]byte(" "), blockLen+10), search),
            want:    1034,
        },
        test{
            search:  search,
            sep:     []byte("\n"),
            readLen: blockLen,
            input:   fmt.Sprintf("%s%stail content", bytes.Repeat([]byte(" "), (blockLen*2)+10), search),
            want:    2058,
        },
        test{
            search:  search,
            sep:     []byte("\n"),
            readLen: blockLen,
            input:   fmt.Sprintf("%s%s%stail content", bytes.Repeat([]byte(" "), (blockLen*2)+10), search, search),
            want:    2065,
        },
    }

    for i, test := range tests {
        s, err := bytesSearch([]byte(test.input))
        if err != nil {
            t.Fatal(err)
        }
        s.Buffer = make([]byte, test.readLen)
        s.Sep = test.sep
        got, err := s.Index(test.search)
        if err != nil {
            t.Fatal(err)
        }
        if got != test.want {
            t.Fatalf("invalid index at %v got %v wanted %v", i, got, test.want)
        }
        got, err = s.Index2(test.search)
        if err != nil {
            t.Fatal(err)
        }
        if got != test.want {
            t.Fatalf("invalid index at %v got %v wanted %v", i, got, test.want)
        }
    }

}

bench_test.go

package main

import (
    "bytes"
    "fmt"
    "testing"

    "github.com/mattetti/filebuffer"
)

func BenchmarkIndex(b *testing.B) {
    search := []byte("find me")
    blockLen := 1024
    input := fmt.Sprintf("%s%stail content", bytes.Repeat([]byte(" "), blockLen-5), search)
    input += input
    s := tailSearch{}
    s.F = filebuffer.New([]byte(input))
    s.Buffer = make([]byte, blockLen)
    for i := 0; i < b.N; i++ {
        s.start = int64(len(input))
        _, err := s.Index(search)
        if err != nil {
            b.Fatal(err)
        }
    }
}

func BenchmarkIndex2(b *testing.B) {
    search := []byte("find me")
    blockLen := 1024
    input := fmt.Sprintf("%s%stail content", bytes.Repeat([]byte(" "), blockLen-5), search)
    input += input
    s := tailSearch{}
    s.F = filebuffer.New([]byte(input))
    s.Buffer = make([]byte, blockLen)
    for i := 0; i < b.N; i++ {
        s.start = int64(len(input))
        _, err := s.Index2(search)
        if err != nil {
            b.Fatal(err)
        }
    }
}

测试

$ go test -v
=== RUN   TestOne
--- PASS: TestOne (0.00s)
PASS
ok      test/backwardsearch 0.002s
$ go test -bench=. -benchmem -v
=== RUN   TestOne
--- PASS: TestOne (0.00s)
goos: linux
goarch: amd64
pkg: test/backwardsearch
BenchmarkIndex-4        20000000           108 ns/op           0 B/op          0 allocs/op
BenchmarkIndex2-4       10000000           167 ns/op           0 B/op          0 allocs/op
PASS
ok      test/backwardsearch 4.129s
$ echo "rrrrfindme" | go run main.go -v
2019/10/17 12:17:04 a start 11
2019/10/17 12:17:04 f n 11 err <nil>
2019/10/17 12:17:04 g start 0
Index  4
2019/10/17 12:17:04 a start 11
2019/10/17 12:17:04 f n 11 err <nil>
2019/10/17 12:17:04 g start 0
2019/10/17 12:17:04 g start 0
Index  4
$ cat bench_test.go | go run main.go -search main
Index  8
Index  8
$ go run main.go 
Index  2056
Index  2056

Answer 2

请注意，我误读了您的问题，并认为它是关于从字符串中读取的。我更新了我的答案，但我也喜欢我的字符串阅读方法。因此，我会把它留在这里。前往下方查看新答案。

如果特殊字符始终相同，则解决问题的例程很简单。

第一个：查找该特殊字符字符串的索引。
第二：如果找到，将该索引添加到该特殊字符字符串的长度。
第三：然后获取所有内容（特殊字符字符串的索引+特殊字符字符串的长度）到内容的末尾。

package main

import "fmt"
import "strings"

func main() {
    var str = `
this is the very large 
part, contains lots lines.

#SPECIAL CHARS START#
a
...
this is the small part the the end, 
contain several lines, but 
we do not know how many lines
this part contains
`   
    var specialStr = "#SPECIAL CHARS START#";
    var lengthOfSpecial = len(specialStr);
    var indexOf = strings.Index(str, specialStr);
    var contentAfter string;

    if ( indexOf != -1 ){
        // If found get content
        indexOf += lengthOfSpecial;
        contentAfter = str[indexOf:];    
    } else {
        // If not content after empty.
        contentAfter = "";
    }

    fmt.Print(contentAfter);
}

---

新答案

抱歉，这么长时间没有写太多 Golang 了。因此，我只能想出下面的代码。我什至忘记了你不需要“;”在行尾:)。

就像已经提到的其他答案一样。从向后读取文件并不是什么新鲜事，它只是你如何做到的。当涉及到代码时，它非常简单。

您从文件的后面开始，将文件的每个块读取到定义的大小。然后搜索那个特殊的字符。如果找不到特殊字符字符串，则将结果保存到数据保存切片中并继续。如果你找到了那个特殊的字符字符串，那么你只会在需要找到字符串并保存到保存切片之后得到什么。然后从读取循环中跳出。完成所有操作后，加入并以字符串形式返回保存切片的数据。

听起来很简单？有一个问题以及我是如何处理的，我在该部分的代码注释中写了非常详细的内容。你必须考虑在那里做什么。那么问题是什么？ 如果您逐块读取文件，预计您要查找的内容可能会被分成两个块。因此，必须考虑到这一点。

请注意，我使用字节切片并直接以字节为单位。其次，字节持有分片索引大小组成，它的索引从高到低使用，因为你不想追加然后反转它。另外，“perRead”的大小不能小于查找字符串的长度。

另外，你想如何处理错误？是否退出、返回对偶值等...由您决定。另外，请注意，我假设您要阅读的所有文件都具有特殊字符。否则，您可以简单地跟踪是否在文件中找到了特殊字符字符串。如果不只是返回一个空字符串。如果找到，则编写并返回结果字符串。

请注意，我最终找到了一种解决方案，可以从两个连续的读取块中正确检查一小块。我在“var halfpart”声明之前将其记录到代码中。

package main

import "fmt"
import "os"
import "bytes"

func getLog(fileName string, findStr string) string {
  const perRead int64 = 512

  file, err := os.Open(fileName)
  if err != nil {
    // error code go here for open file error.
      os.Exit(1)
  }
  stat, err := file.Stat()
  if err != nil {
    // error code go here for getting file stat.
      os.Exit(1)
  }

  // Convert specialChar to find to bytes for fast searching.
  var findBytes = []byte(findStr)
  var findLength = len(findBytes)
  // The length of findStr can't be larger than a read.
  if int64(findLength) > perRead { os.Exit(1) }

  var lastRead = stat.Size()
  var contents = make([][]byte, lastRead / perRead + 1)
  var lastIndex = len(contents) - 1
  var saveIndex = lastIndex

  for {
    var readBytes []byte

    if ( lastRead == 0 ){ break }
    if ( lastRead - perRead > -1 ){
      readBytes =  make([]byte, perRead )
      lastRead = lastRead - perRead
    } else {
      readBytes = make([]byte, lastRead - 0)
      lastRead = 0
    }

    _, err = file.ReadAt(readBytes, lastRead)
    if ( err != nil ){
      // error code go here for reading error
      // This method can't never encounter an eof error
      os.Exit(1);
    }

    var indexOf = bytes.Index(readBytes, findBytes)

    if indexOf != -1 {
      contents[saveIndex] = readBytes[indexOf + findLength:]
      saveIndex -= 1
      break
    } else {
      if saveIndex < lastIndex {
        // So for here, take a small chunk of the beginning of last found(equal to findStr's length) 
        // add to a small ended chunk of this found(equal to findStr's length)
        // However, if this found is less than findStr length,// Then grab whatever available.
        var halfpart []byte
        if len(readBytes) < findLength {
          halfpart = append(readBytes, contents[saveIndex + 1][:findLength]...)
        } else {
          halfpart = append(readBytes[len(readBytes) - findLength:], contents[saveIndex + 1][:findLength]...)
        }

        var indexOf2 = bytes.Index(halfpart, findBytes)
        if indexOf2 != -1 {
          saveIndex = saveIndex + 1
          contents[saveIndex] = append(halfpart[indexOf2 + findLength:], contents[saveIndex][findLength:]...)
          saveIndex -= 1
          break
        }
      }
      contents[saveIndex] = readBytes
      saveIndex -= 1
    }
  }

  for i := saveIndex; i > -1; i-- {
    contents[saveIndex] = []byte{}
  }

  return string(bytes.Join(contents,[]byte{}))
}

func main() {
  var fileName = "test.txt"
  var findStr = "#SPECIAL CHARS START#"
  fmt.Println(getLog(fileName, findStr))
}

test.txt 内容：

Note that, I misread your question and thought that it was about reading from string. I will update this answer tomorrow.

The routines to solve your problem is simple if the special chars are always the same.

1st: Look for the Index of that special chars string.
2nd: If found, add that index to the length of that special chars string.
3rd: Then get all the content from that (index of special chars string + length of special chars string) to the end of the content.
#SPECIAL CHARS START#
The header lines were kept separate because they looked like mail
headers and I have mailmode on.  The same thing applies to Bozo's
quoted text.  Mailmode doesn't screw things up very often, but since
most people are usually converting non-mail, it's off by default.

Paragraphs are handled ok.  In fact, this one is here just to
demonstrate that.

THIS LINE IS VERY IMPORTANT!
(Ok, it wasn't *that* important)


EXAMPLE HEADER
==============

Since this is the first header noticed (all caps, underlined with an
"="), it will be a level 1 header.  It gets an anchor named
"section_1".