投稿
  1. 极客之音首页
  2. 技术随笔

go操作亚马逊S3教程(aws)

飞熊 • • 技术随笔 • 阅读 357

有时候，不是因为你没有能力，也不是因为你缺少勇气，只是因为你付出的努力还太少，所以，成功便不会走向你。而你所需要做的，就是坚定你的梦想，你的目标，你的未来，然后以不达目的誓不罢休的那股劲，去付出你的努力，成功就会慢慢向你靠近。

导读：本篇文章讲解 go操作亚马逊S3教程(aws)，希望对大家有帮助，欢迎收藏，转发！站点地址：www.bmabk.com，来源：原文

go操作亚马逊S3教程

官网API地址：https://docs.aws.amazon.com/sdk-for-go/api/service/s3/

前置：S3介绍

一、aws-sdk-go

1 初始化S3配置

初始化session和连接，同时打印错误日志

var (
	sess *session.Session
	svc  *s3.S3
)

func init() {
	access_key := "xxxxxx"
	secret_key := "xxxxxxxxxxx"
	end_point := "http://xxx.xxx.xxx.xxx:xxx/" //根据自己endpint修改

	sess, _ = session.NewSession(&aws.Config{
		Credentials:      credentials.NewStaticCredentials(access_key, secret_key, ""),
		Endpoint:         aws.String(end_point),
		Region:           aws.String("us-east-1"),
		DisableSSL:       aws.Bool(true),
		S3ForcePathStyle: aws.Bool(false), //virtual-host style方式，不要修改
	})

	svc = s3.New(sess)
}

func main(){
	....
}
func exitErrorf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, msg+"\n", args...)
	os.Exit(1)
}

2 具体操作

2.1 获取所有桶信息

//获取所有桶
func ListBuckets() {
	result, err := svc.ListBuckets(nil)
	if err != nil {
		exitErrorf("Unable to list buckets, %v", err)
	}

	fmt.Println("Buckets:")
	for _, b := range result.Buckets {
		fmt.Printf("* %s created on %s\n",
			aws.StringValue(b.Name), aws.TimeValue(b.CreationDate))
	}

	for _, b := range result.Buckets {
		fmt.Printf("%s\n", aws.StringValue(b.Name))
	}
}

2.2 创建桶

func createBucket(bucket string) {
	params := &s3.CreateBucketInput{
		Bucket: aws.String(bucket),
	}

	_, err := svc.CreateBucket(params)

	if err != nil {
		exitErrorf("Unable to create bucket %q, %v", bucket, err)
	}

	// Wait until bucket is created before finishing
	fmt.Printf("Waiting for bucket %q to be created...\n", bucket)

	err = svc.WaitUntilBucketExists(&s3.HeadBucketInput{
		Bucket: aws.String(bucket),
	})

	if err != nil {
		exitErrorf("Error occurred while waiting for bucket to be created, %v", bucket)
	}

	fmt.Printf("Bucket %q successfully created\n", bucket)
}

2.3 删除桶

func deleteBucket(bucket string) {
	params := &s3.DeleteBucketInput{
		Bucket: aws.String(bucket),
	}

	_, err := svc.DeleteBucket(params)

	if err != nil {
		exitErrorf("Unable to delete bucket %q, %v", bucket, err)
	}

	//wait until bucket is deleted
	err = svc.WaitUntilBucketNotExists(&s3.HeadBucketInput{
		Bucket: aws.String(bucket),
	})

	if err != nil {
		exitErrorf("Error occurred while waiting for bucket to be deleted, %v", bucket)
	}

	fmt.Printf("Bucket %q successfully delete\n", bucket)
}

2.4 下载桶里的文件

func downloadFile(bucket, item string) {
	file, err := os.Create(item)
	if err != nil {
		exitErrorf("Unable to open file %q, %v", err)
	}

	defer file.Close()

	downloader := s3manager.NewDownloader(sess)

	numBytes, err := downloader.Download(file,
		&s3.GetObjectInput{
			Bucket: aws.String(bucket),
			Key:    aws.String(item),
		})
	if err != nil {
		exitErrorf("Unable to download item %q, %v", item, err)
	}

	fmt.Println("Downloaded", file.Name(), numBytes, "bytes")
}

2.5 删除桶里的文件

func deleteFile(bucket, obj string) {
	_, err := svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(obj)})
	if err != nil {
		exitErrorf("Unable to delete object %q from bucket %q, %v", obj, bucket, err)
	}

	err = svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(obj),
	})

	fmt.Printf("Object %q successfully deleted\n", obj)
}

2.6 上传文件到指定桶

func uploadFile(bucket, filename string) {
	file, err := os.Open(filename)
	if err != nil {
		exitErrorf("Unable to open file %q, %v", err)
	}

	defer file.Close()

	uploader := s3manager.NewUploader(sess)

	_, err = uploader.Upload(&s3manager.UploadInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(filename),
		Body:   file,
	})
	if err != nil {
		// Print the error and exit.
		exitErrorf("Unable to upload %q to %q, %v", filename, bucket, err)
	}

	fmt.Printf("Successfully uploaded %q to %q\n", filename, bucket)
}

3 全部代码

package main

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	_ "github.com/aws/aws-sdk-go/service/s3/s3manager"
	"os"
)

var (
	sess *session.Session
	svc  *s3.S3
)

func init() {
	access_key := "xxxxxx"
	secret_key := "xxxxxxxxxxx"
	end_point := "http://xxx.xxx.xxx.xxx:xxx/" //根据自己endpint修改

	sess, _ = session.NewSession(&aws.Config{
		Credentials:      credentials.NewStaticCredentials(access_key, secret_key, ""),
		Endpoint:         aws.String(end_point),
		Region:           aws.String("us-east-1"),
		DisableSSL:       aws.Bool(true),
		S3ForcePathStyle: aws.Bool(false), //virtual-host style方式，不要修改
	})

	svc = s3.New(sess)
}

func main() {
	//注意！！！桶名称后面一定要根`/`
	//downloadFile("bucket-ziyi/", "banner.png")
	//uploadFile("bucket-ziyi/", "test/test1/test2/test.csv")
	//createBuckt("bucket-demo1/")
	//deleteBucket("bucket-demo1/")

	deleteFile("bucket-ziyi/test/", "log.txt")

}

func deleteFile(bucket, obj string) {
	_, err := svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(obj)})
	if err != nil {
		exitErrorf("Unable to delete object %q from bucket %q, %v", obj, bucket, err)
	}

	err = svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(obj),
	})

	fmt.Printf("Object %q successfully deleted\n", obj)
}

func deleteBucket(bucket string) {
	params := &s3.DeleteBucketInput{
		Bucket: aws.String(bucket),
	}

	_, err := svc.DeleteBucket(params)

	if err != nil {
		exitErrorf("Unable to delete bucket %q, %v", bucket, err)
	}

	//wait until bucket is deleted
	err = svc.WaitUntilBucketNotExists(&s3.HeadBucketInput{
		Bucket: aws.String(bucket),
	})

	if err != nil {
		exitErrorf("Error occurred while waiting for bucket to be deleted, %v", bucket)
	}

	fmt.Printf("Bucket %q successfully delete\n", bucket)
}

func createBucket(bucket string) {
	params := &s3.CreateBucketInput{
		Bucket: aws.String(bucket),
	}

	_, err := svc.CreateBucket(params)

	if err != nil {
		exitErrorf("Unable to create bucket %q, %v", bucket, err)
	}

	// Wait until bucket is created before finishing
	fmt.Printf("Waiting for bucket %q to be created...\n", bucket)

	err = svc.WaitUntilBucketExists(&s3.HeadBucketInput{
		Bucket: aws.String(bucket),
	})

	if err != nil {
		exitErrorf("Error occurred while waiting for bucket to be created, %v", bucket)
	}

	fmt.Printf("Bucket %q successfully created\n", bucket)
}

func uploadFile(bucket, filename string) {
	file, err := os.Open(filename)
	if err != nil {
		exitErrorf("Unable to open file %q, %v", err)
	}

	defer file.Close()

	uploader := s3manager.NewUploader(sess)

	_, err = uploader.Upload(&s3manager.UploadInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(filename),
		Body:   file,
	})
	if err != nil {
		// Print the error and exit.
		exitErrorf("Unable to upload %q to %q, %v", filename, bucket, err)
	}

	fmt.Printf("Successfully uploaded %q to %q\n", filename, bucket)
}

func downloadFile(bucket, item string) {
	file, err := os.Create(item)
	if err != nil {
		exitErrorf("Unable to open file %q, %v", err)
	}

	defer file.Close()

	downloader := s3manager.NewDownloader(sess)

	numBytes, err := downloader.Download(file,
		&s3.GetObjectInput{
			Bucket: aws.String(bucket),
			Key:    aws.String(item),
		})
	if err != nil {
		exitErrorf("Unable to download item %q, %v", item, err)
	}

	fmt.Println("Downloaded", file.Name(), numBytes, "bytes")
}

//获取所有桶
func ListBuckets() {
	result, err := svc.ListBuckets(nil)
	if err != nil {
		exitErrorf("Unable to list buckets, %v", err)
	}

	fmt.Println("Buckets:")
	for _, b := range result.Buckets {
		fmt.Printf("* %s created on %s\n",
			aws.StringValue(b.Name), aws.TimeValue(b.CreationDate))
	}

	for _, b := range result.Buckets {
		fmt.Printf("%s\n", aws.StringValue(b.Name))
	}
}

func exitErrorf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, msg+"\n", args...)
	os.Exit(1)
}

二、aws-sdk-go-v2

package main

import (
    "archive/zip"
    "bufio"
    "context"
    "fmt"
    "github.com/aobco/log"
    "github.com/aws/aws-sdk-go-v2/aws"
    "github.com/aws/aws-sdk-go-v2/credentials"
    "github.com/aws/aws-sdk-go-v2/service/s3"
    "io"
    "os"
    "path/filepath"
    "strings"
)

var (
    ak       = "xxxxxxxxxxxxxxx"
    sk       = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
    endpoint = "http://xxx.xxx.xxx:xx/" //endpoint设置，不要动
    client   *s3.Client
)

func init() {
    // 配置AWS SDK
    var credentialsProvider aws.CredentialsProvider
    credentialsProvider = credentials.StaticCredentialsProvider{Value: aws.Credentials{AccessKeyID: ak, SecretAccessKey: sk, SessionToken: ""}}
    cfg := aws.Config{
       Credentials: credentialsProvider,
       EndpointResolverWithOptions: aws.EndpointResolverWithOptionsFunc(
          func(service string, region string, options ...interface{}) (aws.Endpoint, error) {
             return aws.Endpoint{
                URL:               endpoint,
                HostnameImmutable: true,
             }, nil
          }),
    }

    // 创建S3客户端
    client = s3.NewFromConfig(cfg)
}
func main() {
    //step1 : download target file
    key1 := "log1.txt"
    key2 := "log/log2.txt"
    bucket1 := "bucket-ziyi"
    bucket2 := "bucket-ziyi"
    tmpDir := "test/test1/test2/tmp"
    DownloadFile(bucket1, key1, fmt.Sprintf("%v/%v", tmpDir, key1))
    DownloadFile(bucket2, key2, fmt.Sprintf("%v/%v", tmpDir, key2))

    //step2 : compression
    zipFileName, err := compressionDir(tmpDir)
    if err != nil {
       log.Fatal("compression file err=", err)
       return
    }
    //fmt.Println(zipFileName)

    //step3 : upload zip
    key3 := "demo.zip"
    bucket3 := "bucket-ziyi"
    UploadFile(bucket3, key3, zipFileName)
}

// UploadFile 上传文件到S3
func UploadFile(bucket string, key string, filePath string) {
    // 打开本地文件
    file, err := os.Open(filePath)
    if err != nil {
       log.Info("upload file err=", err)
    }
    defer file.Close()
    // 上传文件到S3 bucket
    _, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
       Bucket: &bucket,
       Key:    &key,
       Body:   file,
    })
    if err != nil {
       log.Error("upload file err=", err)
       return
    }
    log.Infof("upload file success, bucket=%v, name=%v", bucket, key)
}

// DownloadFile 下载S3文件到本地
func DownloadFile(bucket string, key string, filePath string) {
    // 下载文件内容
    resp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
       Bucket: &bucket,
       Key:    &key,
    })
    if err != nil {
       log.Error("download file err=", err)
       return
    }
    defer resp.Body.Close()

    index := strings.LastIndex(filePath, "/")
    dir := filePath[:index]
    err = os.MkdirAll(dir, os.ModePerm)
    if err != nil {
       log.Error("download file err=", err)
    }
    // 写入文件
    file, err := os.Create(filePath)
    if err != nil {
       log.Error("download file err=", err)
       return
    }
    defer file.Close()
    buffer := make([]byte, 1024*2)
    for {
       reader := bufio.NewReader(resp.Body)
       n, err := reader.Read(buffer)
       if err == io.EOF {
          break
       }
       if err != nil {
          log.Fatal("download file err=", err)
          return
       }
       file.Write(buffer[:n])
    }
    log.Info("download file success, filePath=", filePath)
}

func compressionDir(baseDir string) (string, error) {
    zipFileName := baseDir + ".zip"

    // 创建一个新的 zip 文件
    zipFile, err := os.Create(zipFileName)
    if err != nil {
       return "", err
    }
    defer zipFile.Close()

    // 创建一个 zip.Writer
    zipWriter := zip.NewWriter(zipFile)
    defer zipWriter.Close()

    // 遍历目录下的所有文件和子目录
    err = filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
       if err != nil {
          return err
       }

       // 创建一个 zip 文件中的文件或目录
       relativePath := strings.TrimPrefix(path, baseDir)
       zipPath := strings.TrimLeft(filepath.Join("/", relativePath), "/")

       // 如果是目录或空目录，则在 zip 文件中创建一个目录
       if info.IsDir() || isEmptyDir(path) {
          _, err := zipWriter.Create(zipPath + "/")
          if err != nil {
             return err
          }
       } else {
          // 如果是文件，则创建一个 zip 文件中的文件
          zipFile, err := zipWriter.Create(zipPath)
          if err != nil {
             return err
          }

          // 打开原始文件
          file, err := os.Open(path)
          if err != nil {
             return err
          }
          defer file.Close()

          // 将原始文件的内容拷贝到 zip 文件中
          _, err = io.Copy(zipFile, file)
          if err != nil {
             return err
          }
       }

       return nil
    })

    if err != nil {
       return "", err
    }
    return zipFileName, nil
}

// 判断目录是否为空目录
func isEmptyDir(dirPath string) bool {
    dir, err := os.Open(dirPath)
    if err != nil {
       return false
    }
    defer dir.Close()

    _, err = dir.Readdirnames(1)
    return err == io.EOF
}

三、aws相关bug

①Versions bucket is not support origin.

对应bucket没有开版本控制

版权声明：本文内容由互联网用户自发贡献，该文观点仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 举报，一经查实，本站将立刻删除。

文章由极客之家整理，本文链接：https://www.bmabk.com/index.php/post/148486.html

赞 (0)
飞熊的头像飞熊bm
0 0

相关推荐

发表回复

登录后才能评论
极客之家——专业性很强的中文编程技术网站，欢迎收藏到浏览器，订阅我们！