go-ayo/common을 gocommon으로 분리
This commit is contained in:
392
s3/func.go
Normal file
392
s3/func.go
Normal file
@ -0,0 +1,392 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
HMAC_ALGORITHM = "HmacSHA256"
|
||||
AWS_ALGORITHM = "AWS4-HMAC-SHA256"
|
||||
SERVICE_NAME = "s3"
|
||||
REQUEST_TYPE = "aws4_request"
|
||||
UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"
|
||||
REGION_NAME = "kr-standard"
|
||||
ENDPOINT = "https://kr.object.ncloudstorage.com"
|
||||
)
|
||||
|
||||
type sortedHeader map[string]string
|
||||
|
||||
var errAccessKeyIsMissing = errors.New("NCLOUD_ACCESS_KEY is missing")
|
||||
var errSecretKeyIsMissing = errors.New("NCLOUD_SECRET_KEY is missing")
|
||||
var errRegionIsMissing = errors.New("NCLOUD_REGION is missing")
|
||||
|
||||
func sortVersions(versions []string) []string {
|
||||
sort.Slice(versions, func(i, j int) bool {
|
||||
leftnum := 0
|
||||
for _, iv := range strings.Split(versions[i], ".") {
|
||||
n, _ := strconv.Atoi(iv)
|
||||
leftnum += leftnum<<8 + n
|
||||
}
|
||||
|
||||
rightnum := 0
|
||||
for _, iv := range strings.Split(versions[j], ".") {
|
||||
n, _ := strconv.Atoi(iv)
|
||||
rightnum += rightnum<<8 + n
|
||||
}
|
||||
|
||||
return leftnum < rightnum
|
||||
})
|
||||
return versions
|
||||
}
|
||||
|
||||
func sign(data string, key []byte) []byte {
|
||||
mac := hmac.New(sha256.New, key)
|
||||
mac.Write([]byte(data))
|
||||
return mac.Sum(nil)
|
||||
}
|
||||
|
||||
func hash(text string) string {
|
||||
s256 := sha256.New()
|
||||
s256.Write([]byte(text))
|
||||
return hex.EncodeToString(s256.Sum(nil))
|
||||
}
|
||||
|
||||
func getStandardizedQueryParameters(query url.Values) string {
|
||||
return query.Encode()
|
||||
}
|
||||
|
||||
func getSignedHeaders(header http.Header) string {
|
||||
keys := make([]string, 0, len(header))
|
||||
for k := range header {
|
||||
keys = append(keys, strings.ToLower(k))
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return strings.Join(keys, ";") + ";"
|
||||
}
|
||||
|
||||
func getStandardizedHeaders(header http.Header) string {
|
||||
keys := make([]string, 0, len(header))
|
||||
for k := range header {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
standardHeaders := make([]string, 0, len(header))
|
||||
for _, k := range keys {
|
||||
standardHeaders = append(standardHeaders, fmt.Sprintf("%s:%s", strings.ToLower(k), header.Get(k)))
|
||||
}
|
||||
return strings.Join(standardHeaders, "\n") + "\n"
|
||||
}
|
||||
|
||||
func getCanonicalRequest(req *http.Request, standardizedQueryParam string, standardHeaders string, signedHeader string) string {
|
||||
return strings.Join([]string{
|
||||
req.Method,
|
||||
req.URL.Path,
|
||||
standardizedQueryParam,
|
||||
standardHeaders,
|
||||
signedHeader,
|
||||
UNSIGNED_PAYLOAD,
|
||||
}, "\n")
|
||||
}
|
||||
|
||||
func getScope(datestamp string, regionName string) string {
|
||||
return strings.Join([]string{
|
||||
datestamp,
|
||||
regionName, // "kr-standard"
|
||||
SERVICE_NAME,
|
||||
REQUEST_TYPE,
|
||||
}, "/")
|
||||
}
|
||||
|
||||
func getStringToSign(timestamp string, scope string, canonicalReq string) string {
|
||||
return strings.Join([]string{
|
||||
AWS_ALGORITHM, // AWS_ALGORITHM
|
||||
timestamp,
|
||||
scope,
|
||||
hash(canonicalReq),
|
||||
}, "\n")
|
||||
}
|
||||
|
||||
func getSignature(secretKey string, datestamp string, regionName string, stringToSign string) string {
|
||||
kSecret := []byte("AWS4" + secretKey)
|
||||
kDate := sign(datestamp, kSecret)
|
||||
kRegion := sign(regionName, kDate)
|
||||
kService := sign(SERVICE_NAME, kRegion)
|
||||
signingKey := sign(REQUEST_TYPE, kService)
|
||||
|
||||
return hex.EncodeToString(sign(stringToSign, signingKey))
|
||||
}
|
||||
|
||||
func getAuthorization(accessKey string, scope string, signedHeader string, signature string) string {
|
||||
signingCredentials := accessKey + "/" + scope
|
||||
credential := "Credential=" + signingCredentials
|
||||
signerHeaders := "SignedHeaders=" + signedHeader
|
||||
signatureHeader := "Signature=" + signature
|
||||
|
||||
return fmt.Sprintf("%s %s, %s, %s", AWS_ALGORITHM, credential, signerHeaders, signatureHeader)
|
||||
}
|
||||
|
||||
func (s S3) addAuthorizationHeader(req *http.Request) {
|
||||
req.Header.Add("host", req.Host)
|
||||
|
||||
now := time.Now().UTC()
|
||||
datestamp := now.Format("20060102")
|
||||
timestamp := now.Format("20060102T150405Z")
|
||||
req.Header.Add("x-amz-date", timestamp)
|
||||
req.Header.Add("x-amz-content-sha256", UNSIGNED_PAYLOAD)
|
||||
|
||||
standardizedQueryParameters := getStandardizedQueryParameters(req.URL.Query())
|
||||
signedHeaders := getSignedHeaders(req.Header)
|
||||
standardizedHeaders := getStandardizedHeaders(req.Header)
|
||||
canonicalRequest := getCanonicalRequest(req, standardizedQueryParameters, standardizedHeaders, signedHeaders)
|
||||
scope := getScope(datestamp, s.regionName)
|
||||
stringToSign := getStringToSign(timestamp, scope, canonicalRequest)
|
||||
signature := getSignature(s.secretKey, datestamp, s.regionName, stringToSign)
|
||||
authorization := getAuthorization(s.accessKey, scope, signedHeaders, signature)
|
||||
req.Header.Add("Authorization", authorization)
|
||||
}
|
||||
|
||||
type S3 struct {
|
||||
accessKey string
|
||||
secretKey string
|
||||
regionName string
|
||||
}
|
||||
|
||||
func NewNCloud() (S3, error) {
|
||||
accessKey := os.Getenv("NCLOUD_ACCESS_KEY")
|
||||
if len(accessKey) == 0 {
|
||||
return S3{}, errAccessKeyIsMissing
|
||||
}
|
||||
secretKey := os.Getenv("NCLOUD_SECRET_KEY")
|
||||
if len(secretKey) == 0 {
|
||||
return S3{}, errSecretKeyIsMissing
|
||||
}
|
||||
region := os.Getenv("NCLOUD_REGION")
|
||||
if len(region) == 0 {
|
||||
return S3{}, errRegionIsMissing
|
||||
}
|
||||
return New(accessKey, secretKey, region), nil
|
||||
}
|
||||
|
||||
func New(accessKey string, secretKey string, regionName string) S3 {
|
||||
return S3{
|
||||
accessKey: accessKey,
|
||||
secretKey: secretKey,
|
||||
regionName: regionName,
|
||||
}
|
||||
}
|
||||
|
||||
func (s S3) MakeGetObjectRequest(objectURL string) (*http.Request, error) {
|
||||
req, err := http.NewRequest("GET", objectURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.addAuthorizationHeader(req)
|
||||
|
||||
return req, nil
|
||||
}
|
||||
|
||||
func (s S3) makeGetItemsRequest(prefixURL string, delimiter string) (*http.Request, error) {
|
||||
u, err := url.Parse(prefixURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
endpoint := u.Host
|
||||
relpath := strings.TrimLeft(u.Path, "/")
|
||||
ns := strings.SplitN(relpath, "/", 2)
|
||||
bucket := ns[0]
|
||||
prefix := ""
|
||||
if len(ns) > 1 {
|
||||
prefix = ns[1]
|
||||
}
|
||||
|
||||
var completeurl string
|
||||
if len(delimiter) > 0 {
|
||||
completeurl = fmt.Sprintf("%s://%s/%s?prefix=%s&delimiter=%s", u.Scheme, endpoint, bucket, prefix, delimiter)
|
||||
} else {
|
||||
completeurl = fmt.Sprintf("%s://%s/%s?prefix=%s", u.Scheme, endpoint, bucket, prefix)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("GET", completeurl, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.addAuthorizationHeader(req)
|
||||
|
||||
return req, nil
|
||||
}
|
||||
|
||||
type FileMeta struct {
|
||||
Key string
|
||||
LastModified time.Time
|
||||
}
|
||||
|
||||
type listBucketResult struct {
|
||||
Name string
|
||||
Prefix string
|
||||
Marker string
|
||||
MaxKeys int
|
||||
Delimiter string
|
||||
IsTruncated bool
|
||||
CommonPrefixes []struct {
|
||||
Prefix string
|
||||
}
|
||||
Contents []FileMeta
|
||||
}
|
||||
|
||||
func (s S3) ListFiles(prefixURL string) ([]FileMeta, error) {
|
||||
req, err := s.makeGetItemsRequest(prefixURL, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var result listBucketResult
|
||||
err = xml.Unmarshal(body, &result)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var out []FileMeta
|
||||
for _, c := range result.Contents {
|
||||
if !strings.HasSuffix(c.Key, "/") {
|
||||
c.Key = prefixURL + "/" + path.Base(c.Key)
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s S3) ListFolders(prefixURL string) ([]string, error) {
|
||||
req, err := s.makeGetItemsRequest(prefixURL, "/")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var result listBucketResult
|
||||
err = xml.Unmarshal(body, &result)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
output := make([]string, 0, len(result.CommonPrefixes))
|
||||
for _, prefix := range result.CommonPrefixes {
|
||||
output = append(output, strings.TrimRight(prefix.Prefix, "/"))
|
||||
}
|
||||
|
||||
return sortVersions(output), nil
|
||||
}
|
||||
|
||||
func (s S3) ReadFile(url string) (io.ReadCloser, error) {
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.addAuthorizationHeader(req)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
func (s S3) DownloadFile(url string, outputFile string) error {
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.addAuthorizationHeader(req)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Create the file
|
||||
out, err := os.Create(outputFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
// Write the body to file
|
||||
_, err = io.Copy(out, resp.Body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s S3) UploadFile(url string, content []byte, publicRead bool) error {
|
||||
req, err := http.NewRequest("PUT", url, bytes.NewReader(content))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.addAuthorizationHeader(req)
|
||||
if publicRead {
|
||||
req.Header.Add("x-amz-acl", "public-read")
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
return nil
|
||||
}
|
||||
return io.EOF
|
||||
}
|
||||
|
||||
// https://api.ncloud-docs.com/docs/storage-objectstorage-putobjectacl
|
||||
func (s S3) SetObjectACL(url string, acl string) error {
|
||||
url += "?acl=" + acl
|
||||
req, err := http.NewRequest("PUT", url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.addAuthorizationHeader(req)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
return nil
|
||||
}
|
||||
return io.EOF
|
||||
}
|
||||
Reference in New Issue
Block a user