从 azure 下载 blob 并解压缩它们 - 故障排除代码 [go]

Question

使用 golang 代码从 Azure 存储下载 Blob。 unzip 函数抛出错误： “bytes.Reader.ReadAt：负offsetBlob解压内容”

这是我的代码？有什么可能出错的建议吗？

传递给 readBlob 函数的信息很好 - 使用此代码我可以打印版本读取并打印实际 zip 文件的元素。

func readBlob(client *azblob.Client, ctx context.Context, containerName string, blobName string) {
    // download the blob
    // will want to do this in the worker pool and not in the main loader thread
    get, err := client.DownloadStream(ctx, containerName, blobName, nil)
    handleError(err)

    downloadedData := bytes.Buffer{}
    retryReader := get.NewRetryReader(ctx, &azblob.RetryReaderOptions{})
    _, err = downloadedData.ReadFrom(retryReader)
    handleError(err)

    err = retryReader.Close()
    handleError(err)

    file, err := decompress(blobName, buff)
    if err != nil {
        fmt.Println("Error decompressing blob: ", blobName)
        fmt.Print(err)
    }
}

func decompress(blobName string, downloadedData bytes.Buffer) (string, error) {
    // try to decompress as zip
    zipReader, err := zip.NewReader(bytes.NewReader(downloadedData.Bytes()), int64(len(downloadedData.Bytes())))
    if err != nil {
        return "", err
    }

    for _, f := range zipReader.File {
    // the next line can add uncomment for troubleshooting and DOES print the right file name -
        does not necessary mean the blob download or the synnex of NewReader is
        perfect - but does validate it somewhat
    // fmt.Println("ffile name in zip file: ", f.Name)
        rc, err := f.Open()  // THIS IS where the error is returned
        if err != nil {
            return "", err
        }
        defer rc.Close()

        decompressedData, err := io.ReadAll(rc)
        if err != nil {
            return "", err
        }
        return string(decompressedData), nil // only care about the first file in the zip file
    }

我尝试过的事情（以及许多其他事情）： 添加（和删除）大量调试打印，并且肯定会下载文件 - 解压缩功能肯定可以看到文件名，但它无法“打开”zip 内的文件

我可以直接通过 Azure 存储资源管理器下载文件，然后使用 Windows 资源管理器打开 zip 文件，然后打开 zip 文件内的文件

Answer 1

从 azure 下载 zip blob 后

package main

import (
    "context"
    "fmt"
    "os"

    "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
)

func main() {
    const (
        connectionString = "<connection_string>"
        containerName    = "sample-container"
        blobName         = "ExifToolWrapper-master.zip"
        downloadFile     = "ExifToolWrapper-master.zip"
    )

    // Create a client with the provided connection string
    client, err := azblob.NewClientFromConnectionString(connectionString, nil)
    if err != nil {
        fmt.Println("Error creating client:", err)
        return
    }

    // Create or open a local file where we can download the blob
    file, err := os.Create(downloadFile)
    if err != nil {
        fmt.Println("Error creating downloaded file:", err)
        return
    }
    defer file.Close()

    // Download the blob
    _, err = client.DownloadFile(context.TODO(), containerName, blobName, file, nil)
    if err != nil {
        fmt.Println("Error downloading file:", err)
        return
    }

    fmt.Println("Blob downloaded successfully!")
}

使用下面的代码是使用标准

archive/zip

包解压文件

package main

import (
    "archive/zip"
    "fmt"
    "io"
    "log"
    "os"
    "path/filepath"
    "strings"
)

func unzipSource(source, destination string) error {
    // 1. Open the zip file
    reader, err := zip.OpenReader(source)
    if err != nil {
        return err
    }
    defer reader.Close()

    // 2. Get the absolute destination path
    destination, err = filepath.Abs(destination)
    if err != nil {
        return err
    }

    // 3. Iterate over zip files inside the archive and unzip each of them
    for _, f := range reader.File {
        err := unzipFile(f, destination)
        if err != nil {
            return err
        }
    }

    return nil
}

func unzipFile(f *zip.File, destination string) error {
    // 4. Check if file paths are not vulnerable to Zip Slip
    filePath := filepath.Join(destination, f.Name)
    if !strings.HasPrefix(filePath, filepath.Clean(destination)+string(os.PathSeparator)) {
        return fmt.Errorf("invalid file path: %s", filePath)
    }

    // 5. Create directory tree
    if f.FileInfo().IsDir() {
        if err := os.MkdirAll(filePath, os.ModePerm); err != nil {
            return err
        }
        return nil
    }

    if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {
        return err
    }

    // 6. Create a destination file for unzipped content
    destinationFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
    if err != nil {
        return err
    }
    defer destinationFile.Close()

    // 7. Unzip the content of a file and copy it to the destination file
    zippedFile, err := f.Open()
    if err != nil {
        return err
    }
    defer zippedFile.Close()

    if _, err := io.Copy(destinationFile, zippedFile); err != nil {
        return err
    }
    return nil
}

func main() {
    zipFilePath := "C:\\Users\\ExifToolWrapper-master.zip"
    destinationDir := "C:\\Users\\folder\\env"

    err := unzipSource(zipFilePath, destinationDir)
    if err != nil {
        log.Fatal(err)
    }
}