gocommon/s3/func.go

package s3

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
	"time"
)

const (
	HMAC_ALGORITHM   = "HmacSHA256"
	AWS_ALGORITHM    = "AWS4-HMAC-SHA256"
	SERVICE_NAME     = "s3"
	REQUEST_TYPE     = "aws4_request"
	UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"
	REGION_NAME      = "kr-standard"
	ENDPOINT         = "https://kr.object.ncloudstorage.com"
)

type sortedHeader map[string]string

var errAccessKeyIsMissing = errors.New("NCLOUD_ACCESS_KEY is missing")
var errSecretKeyIsMissing = errors.New("NCLOUD_SECRET_KEY is missing")
var errRegionIsMissing = errors.New("NCLOUD_REGION is missing")

func sortVersions(versions []string) []string {
	sort.Slice(versions, func(i, j int) bool {
		leftnum := 0
		for _, iv := range strings.Split(versions[i], ".") {
			n, _ := strconv.Atoi(iv)
			leftnum += leftnum<<8 + n
		}

		rightnum := 0
		for _, iv := range strings.Split(versions[j], ".") {
			n, _ := strconv.Atoi(iv)
			rightnum += rightnum<<8 + n
		}

		return leftnum < rightnum
	})
	return versions
}

func sign(data string, key []byte) []byte {
	mac := hmac.New(sha256.New, key)
	mac.Write([]byte(data))
	return mac.Sum(nil)
}

func hash(text string) string {
	s256 := sha256.New()
	s256.Write([]byte(text))
	return hex.EncodeToString(s256.Sum(nil))
}

func getStandardizedQueryParameters(query url.Values) string {
	return query.Encode()
}

func getSignedHeaders(header http.Header) string {
	keys := make([]string, 0, len(header))
	for k := range header {
		keys = append(keys, strings.ToLower(k))
	}
	sort.Strings(keys)
	return strings.Join(keys, ";") + ";"
}

func getStandardizedHeaders(header http.Header) string {
	keys := make([]string, 0, len(header))
	for k := range header {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	standardHeaders := make([]string, 0, len(header))
	for _, k := range keys {
		standardHeaders = append(standardHeaders, fmt.Sprintf("%s:%s", strings.ToLower(k), header.Get(k)))
	}
	return strings.Join(standardHeaders, "\n") + "\n"
}

func getCanonicalRequest(req *http.Request, standardizedQueryParam string, standardHeaders string, signedHeader string) string {
	return strings.Join([]string{
		req.Method,
		req.URL.Path,
		standardizedQueryParam,
		standardHeaders,
		signedHeader,
		UNSIGNED_PAYLOAD,
	}, "\n")
}

func getScope(datestamp string, regionName string) string {
	return strings.Join([]string{
		datestamp,
		regionName, // "kr-standard"
		SERVICE_NAME,
		REQUEST_TYPE,
	}, "/")
}

func getStringToSign(timestamp string, scope string, canonicalReq string) string {
	return strings.Join([]string{
		AWS_ALGORITHM, // AWS_ALGORITHM
		timestamp,
		scope,
		hash(canonicalReq),
	}, "\n")
}

func getSignature(secretKey string, datestamp string, regionName string, stringToSign string) string {
	kSecret := []byte("AWS4" + secretKey)
	kDate := sign(datestamp, kSecret)
	kRegion := sign(regionName, kDate)
	kService := sign(SERVICE_NAME, kRegion)
	signingKey := sign(REQUEST_TYPE, kService)

	return hex.EncodeToString(sign(stringToSign, signingKey))
}

func getAuthorization(accessKey string, scope string, signedHeader string, signature string) string {
	signingCredentials := accessKey + "/" + scope
	credential := "Credential=" + signingCredentials
	signerHeaders := "SignedHeaders=" + signedHeader
	signatureHeader := "Signature=" + signature

	return fmt.Sprintf("%s %s, %s, %s", AWS_ALGORITHM, credential, signerHeaders, signatureHeader)
}

func (s S3) addAuthorizationHeader(req *http.Request) {
	req.Header.Add("host", req.Host)

	now := time.Now().UTC()
	datestamp := now.Format("20060102")
	timestamp := now.Format("20060102T150405Z")
	req.Header.Add("x-amz-date", timestamp)
	req.Header.Add("x-amz-content-sha256", UNSIGNED_PAYLOAD)

	standardizedQueryParameters := getStandardizedQueryParameters(req.URL.Query())
	signedHeaders := getSignedHeaders(req.Header)
	standardizedHeaders := getStandardizedHeaders(req.Header)
	canonicalRequest := getCanonicalRequest(req, standardizedQueryParameters, standardizedHeaders, signedHeaders)
	scope := getScope(datestamp, s.regionName)
	stringToSign := getStringToSign(timestamp, scope, canonicalRequest)
	signature := getSignature(s.secretKey, datestamp, s.regionName, stringToSign)
	authorization := getAuthorization(s.accessKey, scope, signedHeaders, signature)
	req.Header.Add("Authorization", authorization)
}

type S3 struct {
	accessKey  string
	secretKey  string
	regionName string
}

func NewNCloud() (S3, error) {
	accessKey := os.Getenv("NCLOUD_ACCESS_KEY")
	if len(accessKey) == 0 {
		return S3{}, errAccessKeyIsMissing
	}
	secretKey := os.Getenv("NCLOUD_SECRET_KEY")
	if len(secretKey) == 0 {
		return S3{}, errSecretKeyIsMissing
	}
	region := os.Getenv("NCLOUD_REGION")
	if len(region) == 0 {
		return S3{}, errRegionIsMissing
	}
	return New(accessKey, secretKey, region), nil
}

func New(accessKey string, secretKey string, regionName string) S3 {
	return S3{
		accessKey:  accessKey,
		secretKey:  secretKey,
		regionName: regionName,
	}
}

func (s S3) MakeGetObjectRequest(objectURL string) (*http.Request, error) {
	req, err := http.NewRequest("GET", objectURL, nil)
	if err != nil {
		return nil, err
	}

	s.addAuthorizationHeader(req)

	return req, nil
}

func (s S3) makeGetItemsRequest(prefixURL string, delimiter string) (*http.Request, error) {
	u, err := url.Parse(prefixURL)
	if err != nil {
		return nil, err
	}

	endpoint := u.Host
	relpath := strings.TrimLeft(u.Path, "/")
	ns := strings.SplitN(relpath, "/", 2)
	bucket := ns[0]
	prefix := ""
	if len(ns) > 1 {
		prefix = ns[1]
	}

	var completeurl string
	if len(delimiter) > 0 {
		completeurl = fmt.Sprintf("%s://%s/%s?prefix=%s&delimiter=%s", u.Scheme, endpoint, bucket, prefix, delimiter)
	} else {
		completeurl = fmt.Sprintf("%s://%s/%s?prefix=%s", u.Scheme, endpoint, bucket, prefix)
	}

	req, err := http.NewRequest("GET", completeurl, nil)
	if err != nil {
		return nil, err
	}
	s.addAuthorizationHeader(req)

	return req, nil
}

type FileMeta struct {
	Key          string
	LastModified time.Time
}

type listBucketResult struct {
	Name           string
	Prefix         string
	Marker         string
	MaxKeys        int
	Delimiter      string
	IsTruncated    bool
	CommonPrefixes []struct {
		Prefix string
	}
	Contents []FileMeta
}

func (s S3) ListFiles(prefixURL string) ([]FileMeta, error) {
	req, err := s.makeGetItemsRequest(prefixURL, "")
	if err != nil {
		return nil, err
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var result listBucketResult
	err = xml.Unmarshal(body, &result)

	if err != nil {
		return nil, err
	}

	var out []FileMeta
	for _, c := range result.Contents {
		if !strings.HasSuffix(c.Key, "/") {
			c.Key = prefixURL + "/" + path.Base(c.Key)
			out = append(out, c)
		}
	}
	return out, nil
}

func (s S3) ListFolders(prefixURL string) ([]string, error) {
	req, err := s.makeGetItemsRequest(prefixURL, "/")
	if err != nil {
		return nil, err
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var result listBucketResult
	err = xml.Unmarshal(body, &result)
	if err != nil {
		return nil, err
	}

	output := make([]string, 0, len(result.CommonPrefixes))
	for _, prefix := range result.CommonPrefixes {
		output = append(output, strings.TrimRight(prefix.Prefix, "/"))
	}

	return sortVersions(output), nil
}

func (s S3) ReadFile(url string) (io.ReadCloser, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	s.addAuthorizationHeader(req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	return resp.Body, nil
}

func (s S3) DownloadFile(url string, outputFile string) error {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}
	s.addAuthorizationHeader(req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// Create the file
	out, err := os.Create(outputFile)
	if err != nil {
		return err
	}
	defer out.Close()

	// Write the body to file
	_, err = io.Copy(out, resp.Body)
	return err
}

func (s S3) UploadFile(url string, content []byte, publicRead bool) error {
	req, err := http.NewRequest("PUT", url, bytes.NewReader(content))
	if err != nil {
		return err
	}
	s.addAuthorizationHeader(req)
	if publicRead {
		req.Header.Add("x-amz-acl", "public-read")
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusOK {
		return nil
	}
	return io.EOF
}

// https://api.ncloud-docs.com/docs/storage-objectstorage-putobjectacl
func (s S3) SetObjectACL(url string, acl string) error {
	url += "?acl=" + acl
	req, err := http.NewRequest("PUT", url, nil)
	if err != nil {
		return err
	}
	s.addAuthorizationHeader(req)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusOK {
		return nil
	}
	return io.EOF
}