NodeJS 如何从 aws s3 存储桶将文件下载到磁盘?

问题描述 投票:0回答:9

我的目标:

显示一个对话框,提示用户保存从 aws 下载的文件。

我的问题:

我目前正在使用 awssum-amazon-s3 创建下载流。但是,我只能设法将文件保存到我的服务器或将其流式传输到命令行...正如您从我的代码中看到的,我的最后一次尝试是尝试手动设置失败的内容处置标头。我无法使用 res.download() 因为标头已设置?

我怎样才能实现我的目标?

我的节点代码:

app.post('/dls/:dlKey', function(req, res, next){
        // download the file via aws s3 here
        var dlKey = req.param('dlKey');

        Dl.findOne({key:dlKey}, function(err, dl){
            if (err) return next(err);
            var files = dl.dlFile;

            var options = {
                BucketName    : 'xxxx',
                ObjectName    : files,
            };

            s3.GetObject(options, { stream : true }, function(err, data) {
                // stream this file to stdout
                fmt.sep();
                data.Headers['Content-Disposition'] = 'attachment';
                console.log(data.Headers);
                data.Stream.pipe(fs.createWriteStream('test.pdf'));
                data.Stream.on('end', function() {
                    console.log('File Downloaded!');
                });
            });
        });

        res.end('Successful Download Post!');
    });

我的角度代码:

$scope.dlComplete = function (dl) {
        $scope.procDownload = true;
        $http({
            method: 'POST',
            url: '/dls/' + dl.dlKey
        }).success(function(data/*, status, headers, config*/) {
            console.log(data);
            $location.path('/#!/success');
        }).error(function(/*data, status, headers, config*/) {
            console.log('File download failed!');
        });
    };

此代码的目的是让用户使用生成的密钥下载一次文件。

node.js amazon-web-services amazon-s3 download disk
9个回答
99
投票

这是在最新版本的 aws-sdk 上使用流式传输的完整代码

var express = require('express');
var app = express();
var fs = require('fs');

app.get('/', function(req, res, next){
    res.send('You did not say the magic word');
});


app.get('/s3Proxy', function(req, res, next){
    // download the file via aws s3 here
    var fileKey = req.query['fileKey'];

    console.log('Trying to download file', fileKey);
    var AWS = require('aws-sdk');
    AWS.config.update(
      {
        accessKeyId: "....",
        secretAccessKey: "...",
        region: 'ap-southeast-1'
      }
    );
    var s3 = new AWS.S3();
    var options = {
        Bucket    : '/bucket-url',
        Key    : fileKey,
    };

    res.attachment(fileKey);
    var fileStream = s3.getObject(options).createReadStream();
    fileStream.pipe(res);
});

var server = app.listen(3000, function () {
    var host = server.address().address;
    var port = server.address().port;
    console.log('S3 Proxy app listening at http://%s:%s', host, port);
});

27
投票

只需从 S3 创建一个 ReadStream,并为您要下载的位置创建 WriteStream

const AWS = require('aws-sdk');
const path = require('path');
const fs = require('fs');

AWS.config.loadFromPath(path.resolve(__dirname, 'config.json'));
AWS.config.update({
  accessKeyId: AWS.config.credentials.accessKeyId,
  secretAccessKey: AWS.config.credentials.secretAccessKey,
  region: AWS.config.region
});

const s3 = new AWS.S3();
const params = {
  Bucket: '<your-bucket>', 
  Key: '<path-to-your-file>'
};
const readStream = s3.getObject(params).createReadStream();
const writeStream = fs.createWriteStream(path.join(__dirname, 's3data.txt'));
readStream.pipe(writeStream);

24
投票

此代码适用于我最新的库:

var s3 = new AWS.S3();
var s3Params = {
    Bucket: 'your bucket',
    Key: 'path/to/the/file.ext'
};
s3.getObject(s3Params, function(err, res) {
    if (err === null) {
       res.attachment('file.ext'); // or whatever your logic needs
       res.send(data.Body);
    } else {
       res.status(500).send(err);
    }
});

13
投票

使用AWS SDK v3

npm install @aws-sdk/client-s3

下载代码

import { GetObjectCommand } from "@aws-sdk/client-s3";
/**
 * download a file from AWS and send to your rest client
 */
app.get('/download', function(req, res, next){
    var fileKey = req.query['fileKey'];

    var bucketParams = {
        Bucket: 'my-bucket-name',
        Key: fileKey,
    };

    res.attachment(fileKey);
    var fileStream = await s3Client.send(new GetObjectCommand(bucketParams));
    // for TS you can add: if (fileStream.Body instanceof Readable)
    fileStream.Body.pipe(res)
});

7
投票

您已经知道解决问题最重要的是什么:您可以将来自 S3 的文件流传输到任何可写流,无论是文件流……还是将发送到客户端的响应流!

s3.GetObject(options, { stream : true }, function(err, data) {
    res.attachment('test.pdf');
    data.Stream.pipe(res);
});

注意使用

res.attachment
将设置正确的标题。您还可以查看有关流和 S3 的此答案


5
投票

为此,我使用

React frontend
node js backend
。前端我用的是axios。我用这个点击按钮下载文件。

==== Node js 后端代码 (AWS S3) ======

//在

GET
方法中我调用了这个函数

    public download = (req: Request, res: Response) => {
    const keyName = req.query.keyName as string;
    if (!keyName) {
        throw new Error('key is undefined');
    }
    const downloadParams: AWS.S3.GetObjectRequest = {
        Bucket: this.BUCKET_NAME,
        Key: keyName
    };

    this.s3.getObject(downloadParams, (error, data) => {
        if (error) {
            return error;
        }
        res.send(data.Body);
        res.end();
    });
};

======React js前端代码========

//此函数处理下载按钮

onClick

  const downloadHandler = async (keyName: string) => {
  const response = await axiosInstance.get( //here use axios interceptors
    `papers/paper/download?keyName=${keyName}`,{
      responseType:'blob', //very very important dont miss (if not downloaded file unsupported to view)
    }
  );
  const url = window.URL.createObjectURL(new Blob([response.data]));
  const link = document.createElement("a");
  link.href = url;
  link.setAttribute("download", "file.pdf"); //change "file.pdf" according to saved name you want, give extension according to filetype
  document.body.appendChild(link);
  link.click();
  link.remove();
};

------ 或者(如果您使用普通的 axios 而不是 axios 拦截器)-----

axios({
   url: 'http://localhost:5000/static/example.pdf',
   method: 'GET',
   responseType: 'blob', // very very important
}).then((response) => {
   const url = window.URL.createObjectURL(new Blob([response.data]));
   const link = document.createElement('a');
   link.href = url;
   link.setAttribute('download', 'file.pdf');
   document.body.appendChild(link);
   link.click();
});

更多请参考以下文章 1.第1条 2.第2条


1
投票

使用express,基于Jushua的答案和https://docs.aws.amazon.com/AmazonS3/latest/userguide/example_s3_GetObject_section.html

  public downloadFeedFile = (req: IFeedUrlRequest, res: Response) => {
    const downloadParams: GetObjectCommandInput = parseS3Url(req.s3FileUrl.replace(/\s/g, ''));
    logger.info("requesting S3 file  " + JSON.stringify(downloadParams));
    const run = async () => {
      try {
        const fileStream = await this.s3Client.send(new GetObjectCommand(downloadParams));
        if (fileStream.Body instanceof Readable){
          fileStream.Body.once('error', err => {
            console.error("Error downloading s3 file")
            console.error(err);
          });

          fileStream.Body.pipe(res);

        }
      } catch (err) {
        logger.error("Error", err);
      }
    };

  run();
  
  };

0
投票

使用 @aws-sdk/client-s3

GetObjectCommand
:

// The following example retrieves an object for an S3 bucket.
const input = {
  "Bucket": "examplebucket",
  "Key": "HappyFace.jpg"
};

const command = new GetObjectCommand(input);
const response = await client.send(command);

/* response ==
{
  "AcceptRanges": "bytes",
  "ContentLength": "3191",
  "ContentType": "image/jpeg",
  "ETag": "\"6805f2cfc46c0f04559748bb039d69ae\"",
  "LastModified": "Thu, 15 Dec 2016 01:19:41 GMT",
  "Metadata": {},
  "TagCount": 2,
  "VersionId": "null"
}
*/
// example id: to-retrieve-an-object-1481827837012

要将正文流式传输到文件,在 Typescript 中:

import { Readable } from 'stream';

if (!res.Body) {
  throw new Error("No body");
}

const writeStream = fs.createWriteStream(localPath);
const readStream = res.Body as Readable;
readStream.pipe(writeStream);

0
投票
MIGHT HELP ANYONE

app.post("/api/posts",upload.single("demo"),async(req,res)=>{
        console.log('req.body',req.body);
        console.log('req.file',req.file);
        console.log('req.file.asliData',req.file.buffer);
        console.log('req.content-type',req.file.mimetype);
        // req.file.buffer sis the actual image and we need to send it to s3
    
        if (!req.file) {
          return res.status(400).send('No file uploaded');
        }
    
        //resize image only make sure image only
        // const buffer = sharp(req.file.buffer).resize({height:1920,width:1080,fit:"contain"}).toBuffer()
    
    
        const params = {
          Bucket:"usman.test.bucket",
          Key:req.file.originalname, //file name what it should be keep it unique
          Body:req.file.buffer,  // what we are passing should be buffer
          ContentType:req.file.mimetype  //type of file gfood practice
        }
        try {
          //Upload to bucket
          const command  = new PutObjectCommand(params)
          const result = await s3.send(command)
          res.status(200).send(`File uploaded to S3 ${result.ETag}`)
        } catch (error) {
          console.error('Error uploading file to S3:', error);
          res.status(500).send('File upload to S3 failed');
        }
    })
    
    app.get("/api/posts/:fileName", async (req, res) => {
      const fileName = req.params.fileName;
      const bucketName = "usman.test.bucket";
    
      try {
        const params = {
          Bucket: bucketName,
          Key: fileName
        };
    
        const command = new GetObjectCommand(params);
        const {Body,ContentType} = await s3.send(command);
    
        if (Body) {
          const contentType = ContentType;
          res.set("Content-Type", contentType);
          // In a regular HTTP response, the Content-Disposition response header is a header indicating if the content is expected to be displayed inline in the browser, that is, as a Web page or as part of a Web page, or as an attachment, that is downloaded and saved locally.
          // When the value is set to attachment, it tells the browser to treat the response as a downloadable file attachment.
          res.set("Content-Disposition", `attachment; filename="${fileName}"`);
    
          // Pipe the S3 response stream directly to the HTTP response
        
    
        // Body from the GetObjectCommand is a readable stream thats we use pipe (pipe read from codeevolutyiuon)
          Body.pipe(res);
    
          console.log(res);
        } else {
          res.status(404).send("File not found");
        }
      } catch (error) {
        console.error("Error retrieving file from S3:", error);
        res.status(500).send("Error retrieving file from S3");
      }
    });
© www.soinside.com 2019 - 2024. All rights reserved.