如何在golang中读取utf16文本文件到字符串？

问题描述 投票：16回答：4

我可以将文件读取为bytes数组

但是当我将它转换为字符串时

它将utf16字节视为ascii

如何正确转换？

package main

import ("fmt"
"os"
"bufio"
)

func main(){
    // read whole the file
    f, err := os.Open("test.txt")
    if err != nil {
        fmt.Printf("error opening file: %v\n",err)
        os.Exit(1)
    }
    r := bufio.NewReader(f)
    var s,b,e = r.ReadLine()
    if e==nil{
        fmt.Println(b)
        fmt.Println(s)
        fmt.Println(string(s))
    }
}

输出：

假

[255 254 91 0 83 0 99 0 114 0 105 0 112 0 116 0 32 0 73 0 110 0 102 0 111 0 93 0 13 0]

脚本信息]

更新：

在我测试了这两个例子后，我已经明确了现在的确切问题。

在Windows中，如果我在行的末尾添加换行符（CR + LF），则将在行中读取CR。因为readline函数无法正确处理unicode（[OD OA] = ok，[OD 00 OA 00] =不行）。

如果readline函数可以识别unicode，它应该理解[OD 00 OA 00]并返回[] uint16而不是[]字节。

所以我认为我不应该使用bufio.NewReader因为它无法读取utf16，我没有看到bufio.NewReader.ReadLine可以接受参数作为标志来指示读取文本是utf8，utf16le / be或utf32。 go库中的unicode文本是否有任何readline函数？

unicode go readline utf-16
4个回答
12
投票

UTF16，UTF8和字节顺序标记由Unicode Consortium：UTF-16 FAQ，UTF-8 FAQ和Byte Order Mark (BOM) FAQ定义。

Issue 4802: bufio: reading lines is too cumbersome

在Go中，从文件中读取行太麻烦了。

人们经常被吸引到bufio.Reader.ReadLine因为它的名字，但它有一个奇怪的签名，返回（line [] byte，isPrefix bool，错误错误），并且需要大量的工作。

ReadSlice和ReadString需要一个分隔符字节，它几乎总是明显且难看的'\ n'，并且还可以返回一行和一个EOF

Revision: f685026a2d38

bufio：新的Scanner界面

基于名为Scanner的新类型，为扫描（可能是文本）数据添加一个新的简单界面。它有自己的内部缓冲，所以即使没有注入bufio.Reader也应该合理有效。输入的格式由“拆分功能”定义，默认情况下分为行。

go1.1beta1 released

您可以从通常的位置下载二进制和源代码分发：https://code.google.com/p/go/downloads/list?q=go1.1beta1

这是一个使用Unicode规则将UTF16文本文件行转换为Go UTF8编码字符串的程序。该代码已经过修改，以利用Go 1.1中新的bufio.Scanner接口。

package main

import (
    "bufio"
    "bytes"
    "encoding/binary"
    "fmt"
    "os"
    "runtime"
    "unicode/utf16"
    "unicode/utf8"
)

// UTF16BytesToString converts UTF-16 encoded bytes, in big or little endian byte order,
// to a UTF-8 encoded string.
func UTF16BytesToString(b []byte, o binary.ByteOrder) string {
    utf := make([]uint16, (len(b)+(2-1))/2)
    for i := 0; i+(2-1) < len(b); i += 2 {
        utf[i/2] = o.Uint16(b[i:])
    }
    if len(b)/2 < len(utf) {
        utf[len(utf)-1] = utf8.RuneError
    }
    return string(utf16.Decode(utf))
}

// UTF-16 endian byte order
const (
    unknownEndian = iota
    bigEndian
    littleEndian
)

// dropCREndian drops a terminal \r from the endian data.
func dropCREndian(data []byte, t1, t2 byte) []byte {
    if len(data) > 1 {
        if data[len(data)-2] == t1 && data[len(data)-1] == t2 {
            return data[0 : len(data)-2]
        }
    }
    return data
}

// dropCRBE drops a terminal \r from the big endian data.
func dropCRBE(data []byte) []byte {
    return dropCREndian(data, '\x00', '\r')
}

// dropCRLE drops a terminal \r from the little endian data.
func dropCRLE(data []byte) []byte {
    return dropCREndian(data, '\r', '\x00')
}

// dropCR drops a terminal \r from the data.
func dropCR(data []byte) ([]byte, int) {
    var endian = unknownEndian
    switch ld := len(data); {
    case ld != len(dropCRLE(data)):
        endian = littleEndian
    case ld != len(dropCRBE(data)):
        endian = bigEndian
    }
    return data, endian
}

// SplitFunc is a split function for a Scanner that returns each line of
// text, stripped of any trailing end-of-line marker. The returned line may
// be empty. The end-of-line marker is one optional carriage return followed
// by one mandatory newline. In regular expression notation, it is `\r?\n`.
// The last non-empty line of input will be returned even if it has no
// newline.
func ScanUTF16LinesFunc(byteOrder binary.ByteOrder) (bufio.SplitFunc, func() binary.ByteOrder) {

    // Function closure variables
    var endian = unknownEndian
    switch byteOrder {
    case binary.BigEndian:
        endian = bigEndian
    case binary.LittleEndian:
        endian = littleEndian
    }
    const bom = 0xFEFF
    var checkBOM bool = endian == unknownEndian

    // Scanner split function
    splitFunc := func(data []byte, atEOF bool) (advance int, token []byte, err error) {

        if atEOF && len(data) == 0 {
            return 0, nil, nil
        }

        if checkBOM {
            checkBOM = false
            if len(data) > 1 {
                switch uint16(bom) {
                case uint16(data[0])<<8 | uint16(data[1]):
                    endian = bigEndian
                    return 2, nil, nil
                case uint16(data[1])<<8 | uint16(data[0]):
                    endian = littleEndian
                    return 2, nil, nil
                }
            }
        }

        // Scan for newline-terminated lines.
        i := 0
        for {
            j := bytes.IndexByte(data[i:], '\n')
            if j < 0 {
                break
            }
            i += j
            switch e := i % 2; e {
            case 1: // UTF-16BE
                if endian != littleEndian {
                    if i > 1 {
                        if data[i-1] == '\x00' {
                            endian = bigEndian
                            // We have a full newline-terminated line.
                            return i + 1, dropCRBE(data[0 : i-1]), nil
                        }
                    }
                }
            case 0: // UTF-16LE
                if endian != bigEndian {
                    if i+1 < len(data) {
                        i++
                        if data[i] == '\x00' {
                            endian = littleEndian
                            // We have a full newline-terminated line.
                            return i + 1, dropCRLE(data[0 : i-1]), nil
                        }
                    }
                }
            }
            i++
        }

        // If we're at EOF, we have a final, non-terminated line. Return it.
        if atEOF {
            // drop CR.
            advance = len(data)
            switch endian {
            case bigEndian:
                data = dropCRBE(data)
            case littleEndian:
                data = dropCRLE(data)
            default:
                data, endian = dropCR(data)
            }
            if endian == unknownEndian {
                if runtime.GOOS == "windows" {
                    endian = littleEndian
                } else {
                    endian = bigEndian
                }
            }
            return advance, data, nil
        }

        // Request more data.
        return 0, nil, nil
    }

    // Endian byte order function
    orderFunc := func() (byteOrder binary.ByteOrder) {
        switch endian {
        case bigEndian:
            byteOrder = binary.BigEndian
        case littleEndian:
            byteOrder = binary.LittleEndian
        }
        return byteOrder
    }

    return splitFunc, orderFunc
}

func main() {
    file, err := os.Open("utf16.le.txt")
    if err != nil {
        fmt.Println(err)
        os.Exit(1)
    }
    defer file.Close()
    fmt.Println(file.Name())

    rdr := bufio.NewReader(file)
    scanner := bufio.NewScanner(rdr)
    var bo binary.ByteOrder // unknown, infer from data
    // bo = binary.LittleEndian // windows
    splitFunc, orderFunc := ScanUTF16LinesFunc(bo)
    scanner.Split(splitFunc)

    for scanner.Scan() {
        b := scanner.Bytes()
        s := UTF16BytesToString(b, orderFunc())
        fmt.Println(len(s), s)
        fmt.Println(len(b), b)
    }
    fmt.Println(orderFunc())

    if err := scanner.Err(); err != nil {
        fmt.Println(err)
    }
}

输出：

utf16.le.txt
15 "Hello, 世界"
22 [34 0 72 0 101 0 108 0 108 0 111 0 44 0 32 0 22 78 76 117 34 0]
0 
0 []
15 "Hello, 世界"
22 [34 0 72 0 101 0 108 0 108 0 111 0 44 0 32 0 22 78 76 117 34 0]
LittleEndian

utf16.be.txt
15 "Hello, 世界"
22 [0 34 0 72 0 101 0 108 0 108 0 111 0 44 0 32 78 22 117 76 0 34]
0 
0 []
15 "Hello, 世界"
22 [0 34 0 72 0 101 0 108 0 108 0 111 0 44 0 32 78 22 117 76 0 34]
BigEndian
12
投票

最新版本的golang.org/x/text/encoding/unicode使得更容易实现这一点，因为它包含unicode.BOMOverride，它将智能地解释BOM。

这是ReadFileUTF16（），它类似于os.ReadFile（），但解码为UTF-16。

package main

import (
    "bytes"
    "fmt"
    "io/ioutil"
    "log"
    "strings"

    "golang.org/x/text/encoding/unicode"
    "golang.org/x/text/transform"
)

// Similar to ioutil.ReadFile() but decodes UTF-16.  Useful when
// reading data from MS-Windows systems that generate UTF-16BE files,
// but will do the right thing if other BOMs are found.
func ReadFileUTF16(filename string) ([]byte, error) {

    // Read the file into a []byte:
    raw, err := ioutil.ReadFile(filename)
    if err != nil {
        return nil, err
    }

    // Make an tranformer that converts MS-Win default to UTF8:
    win16be := unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM)
    // Make a transformer that is like win16be, but abides by BOM:
    utf16bom := unicode.BOMOverride(win16be.NewDecoder())

    // Make a Reader that uses utf16bom:
    unicodeReader := transform.NewReader(bytes.NewReader(raw), utf16bom)

    // decode and print:
    decoded, err := ioutil.ReadAll(unicodeReader)
    return decoded, err
}

func main() {
    data, err := ReadFileUTF16("inputfile.txt")
    if err != nil {
        log.Fatal(err)
    }
    final := strings.Replace(string(data), "\r\n", "\n", -1)
    fmt.Println(final)

}

这是NewScannerUTF16，它与os.Open（）类似，但返回扫描程序。

package main

import (
    "bufio"
    "fmt"
    "log"
    "os"

    "golang.org/x/text/encoding/unicode"
    "golang.org/x/text/transform"
)

type utfScanner interface {
    Read(p []byte) (n int, err error)
}

// Creates a scanner similar to os.Open() but decodes the file as UTF-16.
// Useful when reading data from MS-Windows systems that generate UTF-16BE
// files, but will do the right thing if other BOMs are found.
func NewScannerUTF16(filename string) (utfScanner, error) {

    // Read the file into a []byte:
    file, err := os.Open(filename)
    if err != nil {
        return nil, err
    }

    // Make an tranformer that converts MS-Win default to UTF8:
    win16be := unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM)
    // Make a transformer that is like win16be, but abides by BOM:
    utf16bom := unicode.BOMOverride(win16be.NewDecoder())

    // Make a Reader that uses utf16bom:
    unicodeReader := transform.NewReader(file, utf16bom)
    return unicodeReader, nil
}

func main() {

    s, err := NewScannerUTF16("inputfile.txt")
    if err != nil {
        log.Fatal(err)
    }

    scanner := bufio.NewScanner(s)
    for scanner.Scan() {
        fmt.Println(scanner.Text()) // Println will add back the final '\n'
    }
    if err := scanner.Err(); err != nil {
        fmt.Fprintln(os.Stderr, "reading inputfile:", err)
    }

}

仅供参考：我已将这些功能纳入开源模块并进行了进一步改进。见https://github.com/TomOnTime/utfutil/

4
投票

例如：

package main

import (
        "errors"
        "fmt"
        "log"
        "unicode/utf16"
)

func utf16toString(b []uint8) (string, error) {
        if len(b)&1 != 0 {
                return "", errors.New("len(b) must be even")
        }

        // Check BOM
        var bom int
        if len(b) >= 2 {
                switch n := int(b[0])<<8 | int(b[1]); n {
                case 0xfffe:
                        bom = 1
                        fallthrough
                case 0xfeff:
                        b = b[2:]
                }
        }

        w := make([]uint16, len(b)/2)
        for i := range w {
                w[i] = uint16(b[2*i+bom&1])<<8 | uint16(b[2*i+(bom+1)&1])
        }
        return string(utf16.Decode(w)), nil
}

func main() {
        // Simulated data from e.g. a file
        b := []byte{255, 254, 91, 0, 83, 0, 99, 0, 114, 0, 105, 0, 112, 0, 116, 0, 32, 0, 73, 0, 110, 0, 102, 0, 111, 0, 93, 0, 13, 0}
        s, err := utf16toString(b)
        if err != nil {
                log.Fatal(err)
        }

        fmt.Printf("%q", s)
}

（也是here）

输出：

"[Script Info]\r"
1
投票

这是阅读它的最简单方法：

package main

import (
    "bufio"
    "fmt"
    "log"
    "os"

    "golang.org/x/text/encoding/unicode"
    "golang.org/x/text/transform"
)

func main() {
    file, err := os.Open("./text.txt")
    if err != nil {
        log.Fatal(err)
    }

    scanner := bufio.NewScanner(transform.NewReader(file, unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewDecoder()))
    for scanner.Scan() {
        fmt.Printf(scanner.Text())
    }
}

由于Windows默认使用little-endian命令link，我们使用unicode.UseBOM策略从文本中检索BOM，并使用unicode.LittleEndian作为后备

0
投票

如果你想要任何东西打印成一个字符串，你可以使用fmt.Sprint

package main

import (
    "bufio"
    "fmt"
    "os"
)

func main() {
    // read whole the file
    f, err := os.Open("test.txt")
    if err != nil {
        fmt.Printf("error opening file: %v\n", err)
        os.Exit(1)
    }
    r := bufio.NewReader(f)
    var s, b, e = r.ReadLine()
    if e == nil {
        fmt.Println(b)
        fmt.Println(s)
        fmt.Println(fmt.Sprint(string(s)))
    }
}
最新问题
© www.soinside.com 2019 - 2024. All rights reserved.