ayo/maingate
8
1
Fork 3
Code Issues Pull Requests Projects Releases Wiki Activity

파일 업로드 추가 / 화이트리스트에 권한 종류 추가

Browse Source
This commit is contained in:
mountain
2023-05-28 22:13:10 +09:00
parent 594b9fb700
commit 4347909aab
6 changed files with 427 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
core/api.go
View File

@ -1,13 +1,19 @@
package core
import (
"bytes"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net/http"
"os"
"path"
"sort"
"strconv"
"strings"
"sync/atomic"
"time"
@ -21,6 +27,54 @@ import (
"go.mongodb.org/mongo-driver/mongo/options"
)
type fileDocumentDesc struct {
Service string
Key string
Src string
Link string
Desc string
Extract bool
Timestamp int64
Contents []byte `json:",omitempty"`
}
func (fd *fileDocumentDesc) save() error {
// 새 파일 올라옴
if len(fd.Contents) == 0 {
return nil
}
var destFile string
if fd.Extract {
os.MkdirAll(fd.Link, os.ModePerm)
destFile = path.Join(fd.Link, fd.Src)
} else {
os.MkdirAll(path.Dir(fd.Link), os.ModePerm)
destFile = fd.Link
}
f, err := os.Create(destFile)
if err != nil {
return err
}
defer f.Close()
_, err = io.Copy(f, bytes.NewBuffer(fd.Contents))
if err != nil {
return err
}
if fd.Extract {
switch path.Ext(destFile) {
case ".zip":
err = common.Unzip(destFile)
case ".tar":
err = common.Untar(destFile)
}
}
return err
}
func (caller apiCaller) isGlobalAdmin() bool {
if *noauth {
return true
@ -60,13 +114,18 @@ func (caller apiCaller) writeAccessableServices(w http.ResponseWriter) {
func (caller apiCaller) getAccessableServices() ([]*serviceDescription, []string) {
allservices := caller.mg.services.all()
v, ok := caller.userinfo["email"]
if !ok {
return nil, nil
}
email := v.(string)
_, admin := caller.admins[email]
admin := caller.isGlobalAdmin()
var email string
if !*noauth {
v, ok := caller.userinfo["email"]
if !ok {
return nil, nil
}
email := v.(string)
_, admin = caller.admins[email]
}
var output []*serviceDescription
var editable []string
@ -114,30 +173,79 @@ func (caller apiCaller) isValidUser(service any, category string) (valid bool, a
return svcdesc.isValidAPIUser(category, email), false
}
func (caller apiCaller) filesAPI(w http.ResponseWriter, r *http.Request) error {
serviceid := r.FormValue("service")
if len(serviceid) == 0 {
serviceid = "000000000000"
}
var files []fileDocumentDesc
err := caller.mg.mongoClient.FindAllAs(CollectionFile, bson.M{
"service": serviceid,
}, &files, options.Find().SetProjection(bson.M{
"contents": 0,
}))
if err != nil {
return err
}
enc := json.NewEncoder(w)
return enc.Encode(files)
}
var seq = uint32(0)
func (caller apiCaller) uploadAPI(w http.ResponseWriter, r *http.Request) error {
// file, header, err := r.FormFile("file")
// if err != nil {
// logger.Error(err)
// w.WriteHeader(http.StatusBadRequest)
// return
// }
// defer file.Close()
serviceid := r.FormValue("service")
if len(serviceid) == 0 {
serviceid = "000000000000"
}
// contents, err := io.ReadAll(file)
// if err != nil {
// logger.Error(err)
// w.WriteHeader(http.StatusInternalServerError)
// return
// }
infile, header, err := r.FormFile("file")
if err != nil {
w.WriteHeader(http.StatusBadRequest)
return err
}
defer infile.Close()
// ext := path.Ext(header.Filename)
// if ext == ".zip" {
desc := r.FormValue("desc")
contents, _ := io.ReadAll(infile)
extractstr := r.FormValue("extract")
extract, _ := strconv.ParseBool(extractstr)
// }
var b [5]byte
binary.BigEndian.PutUint32(b[0:4], uint32(time.Now().Unix()))
b[4] = byte(atomic.AddUint32(&seq, 1) % 255)
rf := hex.EncodeToString(b[:])
// // deploys 폴더는 파일시스템 서비스이므로 다운로드 가능
// filename := path.Join("deploys", name, version, name+ext)
return nil
var link string
if extract {
link = path.Join("static", serviceid, rf)
} else {
link = path.Join("static", serviceid, rf, header.Filename)
}
newdoc := fileDocumentDesc{
Contents: contents,
Src: header.Filename,
Timestamp: time.Now().UTC().Unix(),
Extract: extract,
Link: link,
Desc: desc,
Key: rf,
Service: serviceid,
}
_, _, err = caller.mg.mongoClient.UpsertOne(CollectionFile, bson.M{
"service": serviceid,
"key": rf,
}, newdoc)
if err == nil {
newdoc.Contents = nil
enc := json.NewEncoder(w)
enc.Encode(newdoc)
}
return err
}
func (caller apiCaller) whitelistAPI(w http.ResponseWriter, r *http.Request) error {
@ -215,25 +323,6 @@ func (caller apiCaller) whitelistAPI(w http.ResponseWriter, r *http.Request) err
return nil
}
// func (caller apiCaller) divisionAPI(w http.ResponseWriter, r *http.Request, svcid string, divid string) error {
// if r.Method == "PUT" {
// // svcid, divid에 statemeta 설정
// file, header, err := r.FormFile("file")
// if err != nil {
// logger.Error(err)
// w.WriteHeader(http.StatusBadRequest)
// return
// }
// defer file.Close()
// if header.
// stateFile, header, err := r.FormFile("file")
// if err != nil {
// return err
// }
// }
// }
func (caller apiCaller) serviceAPI(w http.ResponseWriter, r *http.Request) error {
mg := caller.mg
queryvals := r.URL.Query()
@ -277,6 +366,10 @@ func (caller apiCaller) serviceAPI(w http.ResponseWriter, r *http.Request) error
}
filter := bson.M{"_id": service.Id}
if len(service.ServiceCode) == 0 {
service.ServiceCode = hex.EncodeToString(service.Id[6:])
}
success, _, err := mg.mongoClient.Update(CollectionService, filter, bson.M{
"$set": &service,
}, options.Update().SetUpsert(true))
@ -511,6 +604,8 @@ func (mg *Maingate) api(w http.ResponseWriter, r *http.Request) {
err = caller.accountAPI(w, r)
} else if strings.HasSuffix(r.URL.Path, "/upload") {
err = caller.uploadAPI(w, r)
} else if strings.HasSuffix(r.URL.Path, "/files") {
err = caller.filesAPI(w, r)
}
if err != nil {

45
core/maingate.go
View File

@ -39,6 +39,7 @@ var (
CollectionAuth = common.CollectionName("auth")
CollectionWhitelist = common.CollectionName("whitelist")
CollectionService = common.CollectionName("service")
CollectionFile = common.CollectionName("file")
CollectionBlock = common.CollectionName("block")
CollectionPlatformLoginToken = common.CollectionName("platform_login_token") //-- 각 플랫폼에 로그인 및 권한 받아오는 과정에 사용하는 Key
CollectionUserToken = common.CollectionName("usertoken")
@ -434,6 +435,18 @@ func (mg *Maingate) prepare(context context.Context) (err error) {
return err
}
if err = mg.mongoClient.MakeIndices(CollectionFile, map[string]bson.D{
"service": {{Key: "service", Value: 1}},
}); err != nil {
return err
}
if err = mg.mongoClient.MakeUniqueIndices(CollectionFile, map[string]bson.D{
"sk": {{Key: "service", Value: 1}, {Key: "key", Value: 1}},
}); err != nil {
return err
}
if err = mg.mongoClient.MakeExpireIndex(CollectionWhitelist, 10); err != nil {
return err
}
@ -482,6 +495,36 @@ func (mg *Maingate) prepare(context context.Context) (err error) {
mg.auths = makeAuthCollection(mg.mongoClient, time.Duration(mg.SessionTTL*int64(time.Second)))
var preall []struct {
Link string `bson:"link"`
Id primitive.ObjectID `bson:"_id"`
}
if err = mg.mongoClient.FindAllAs(CollectionFile, nil, &preall, options.Find().SetProjection(bson.M{
"link": 1,
})); err != nil {
return err
}
for _, pre := range preall {
_, err := os.Stat(pre.Link)
if !os.IsNotExist(err) {
continue
}
logger.Println("saving files :", pre.Link)
var fulldoc fileDocumentDesc
err = mg.mongoClient.FindOneAs(CollectionFile, bson.M{
"_id": pre.Id,
}, &fulldoc)
if err != nil {
return err
}
err = fulldoc.save()
if err != nil {
return err
}
}
go watchAuthCollection(context, mg.auths, mg.mongoClient)
go mg.watchWhitelistCollection(context)
@ -579,7 +622,7 @@ func (mg *Maingate) RegisterHandlers(ctx context.Context, serveMux *http.ServeMu
serveMux.HandleFunc(common.MakeHttpHandlerPattern(prefix, "authorize_sdk", AuthPlatformFirebaseAuth), mg.platform_firebaseauth_authorize_sdk)
go mg.watchServiceCollection(ctx, serveMux, prefix)
go mg.watchFileCollection(ctx, serveMux, prefix)
// fsx := http.FileServer(http.Dir("console"))
// serveMux.Handle("/console/", http.StripPrefix("/console/", fsx))
// logger.Println("console file server open")

213
core/service.go
View File

@ -7,6 +7,7 @@ import (
"fmt"
"io"
"net/http"
"path"
"strings"
"sync/atomic"
"time"
@ -26,16 +27,25 @@ type blockinfo struct {
Reason string
}
type whitelistAuthType = string
const (
whitelistAuthType_Default = whitelistAuthType("")
whitelistAuthType_QA = whitelistAuthType("qa")
)
type whitelistmember struct {
Service string
Email string
Platform string
Desc string
Auth []whitelistAuthType
Expired primitive.DateTime `bson:"_ts,omitempty" json:"_ts,omitempty"`
}
type whitelist struct {
emailptr unsafe.Pointer
qaptr unsafe.Pointer
working int32
}
@ -50,17 +60,33 @@ type usertokeninfo struct {
}
func (wl *whitelist) init(total []whitelistmember) {
next := make(map[string]*whitelistmember)
auths := make(map[string]map[string]*whitelistmember)
for _, member := range total {
next[whitelistKey(member.Email)] = &member
}
all := auths[""]
if all == nil {
all = make(map[string]*whitelistmember)
auths[""] = all
}
all[whitelistKey(member.Email)] = &member
atomic.StorePointer(&wl.emailptr, unsafe.Pointer(&next))
atomic.StoreInt32(&wl.working, 1)
for _, auth := range member.Auth {
spec := auths[auth]
if spec == nil {
spec = make(map[string]*whitelistmember)
auths[auth] = spec
}
spec[whitelistKey(member.Email)] = &member
}
}
all := auths[whitelistAuthType_Default]
atomic.StorePointer(&wl.emailptr, unsafe.Pointer(&all))
qa := auths[whitelistAuthType_QA]
atomic.StorePointer(&wl.qaptr, unsafe.Pointer(&qa))
}
func (wl *whitelist) add(m *whitelistmember) {
ptr := atomic.LoadPointer(&wl.emailptr)
func addToUnsafePointer(to *unsafe.Pointer, m *whitelistmember) {
ptr := atomic.LoadPointer(to)
src := (*map[string]*whitelistmember)(ptr)
next := map[string]*whitelistmember{}
@ -68,11 +94,11 @@ func (wl *whitelist) add(m *whitelistmember) {
next[k] = v
}
next[whitelistKey(m.Email)] = m
atomic.StorePointer(&wl.emailptr, unsafe.Pointer(&next))
atomic.StorePointer(to, unsafe.Pointer(&next))
}
func (wl *whitelist) remove(email string) {
ptr := atomic.LoadPointer(&wl.emailptr)
func removeFromUnsafePointer(from *unsafe.Pointer, email string) {
ptr := atomic.LoadPointer(from)
src := (*map[string]*whitelistmember)(ptr)
next := make(map[string]*whitelistmember)
@ -80,7 +106,21 @@ func (wl *whitelist) remove(email string) {
next[k] = v
}
delete(next, whitelistKey(email))
atomic.StorePointer(&wl.emailptr, unsafe.Pointer(&next))
atomic.StorePointer(from, unsafe.Pointer(&next))
}
func (wl *whitelist) add(m *whitelistmember) {
addToUnsafePointer(&wl.emailptr, m)
for _, auth := range m.Auth {
if auth == whitelistAuthType_QA {
addToUnsafePointer(&wl.qaptr, m)
}
}
}
func (wl *whitelist) remove(email string) {
removeFromUnsafePointer(&wl.emailptr, email)
removeFromUnsafePointer(&wl.qaptr, email)
}
func (wl *whitelist) isMember(email string, platform string) bool {
@ -97,30 +137,46 @@ func (wl *whitelist) isMember(email string, platform string) bool {
return false
}
func (wl *whitelist) hasAuth(email string, platform string, auth whitelistAuthType) bool {
if auth == whitelistAuthType_QA {
ptr := atomic.LoadPointer(&wl.qaptr)
src := *(*map[string]*whitelistmember)(ptr)
if member, exists := src[whitelistKey(email)]; exists {
return member.Platform == platform
}
}
return false
}
type divisionStateName string
const (
DivisionState_Closed = string("closed")
DivisionState_Maintenance = string("maintenance")
DivisionState_RestrictedOpen = string("restricted")
DivisionState_FullOpen = string("open")
DivisionState_Closed = divisionStateName("closed")
DivisionState_Maintenance = divisionStateName("maintenance")
DivisionState_RestrictedOpen = divisionStateName("restricted")
DivisionState_FullOpen = divisionStateName("open")
)
type maintenance struct {
Link string
StartTime primitive.Timestamp
Notice string `bson:"notice"`
StartTimeUTC int64 `bson:"start_unixtime_utc" json:"start_unixtime_utc"`
link string
}
type division struct {
Url string `bson:"url"`
Priority int `bson:"priority"`
State string `bson:"state"`
Maintenance maintenance `bson:"maintenance"`
Url string `bson:"url"`
Priority int `bson:"priority"`
State divisionStateName `bson:"state"`
Maintenance *maintenance `bson:"maintenance"`
}
type serviceDescription struct {
// sync.Mutex
Id primitive.ObjectID `bson:"_id"`
ServiceName string `bson:"service"`
Divisions map[string]division `bson:"divisions"`
Divisions map[string]*division `bson:"divisions"`
ServiceCode string `bson:"code"`
UseWhitelist bool `bson:"use_whitelist"`
Closed bool `bson:"closed"`
@ -175,12 +231,46 @@ func (sh *serviceDescription) readProfile(authtype string, id string, binfo stri
}
func (sh *serviceDescription) prepare(mg *Maingate) error {
div := sh.Divisions
divs := sh.Divisions
if len(sh.ServiceCode) == 0 {
sh.ServiceCode = hex.EncodeToString(sh.Id[6:])
}
divmarshaled, _ := json.Marshal(div)
var closed []string
for dn, div := range divs {
if div.State == DivisionState_Closed {
closed = append(closed, dn)
continue
}
if len(div.State) == 0 {
div.State = DivisionState_FullOpen
}
if div.State != DivisionState_FullOpen {
if div.Maintenance == nil {
div.Maintenance = &maintenance{}
}
if len(div.Maintenance.link) == 0 {
if len(div.Maintenance.Notice) == 0 {
div.Maintenance.link = "https://www.action2quare.com"
} else if strings.HasPrefix(div.Maintenance.Notice, "http") {
div.Maintenance.link = div.Maintenance.Notice
} else {
div.Maintenance.link = path.Join("static", sh.ServiceCode, div.Maintenance.Notice)
}
}
} else {
div.Maintenance = nil
}
}
for _, dn := range closed {
delete(divs, dn)
}
divmarshaled, _ := json.Marshal(divs)
devstr := string(divmarshaled)
sh.divisionsSerialized = unsafe.Pointer(&devstr)
@ -200,15 +290,17 @@ func (sh *serviceDescription) prepare(mg *Maingate) error {
sh.closed = 0
}
if sh.UseWhitelist {
var whites []whitelistmember
if err := mg.mongoClient.FindAllAs(CollectionWhitelist, bson.M{
"$or": []bson.M{{"service": sh.ServiceName}, {"service": sh.ServiceCode}},
}, &whites, options.Find().SetReturnKey(false)); err != nil {
return err
}
var whites []whitelistmember
if err := mg.mongoClient.FindAllAs(CollectionWhitelist, bson.M{
"$or": []bson.M{{"service": sh.ServiceName}, {"service": sh.ServiceCode}},
}, &whites, options.Find().SetReturnKey(false)); err != nil {
return err
}
sh.wl.init(whites)
sh.wl.init(whites)
if sh.UseWhitelist {
sh.wl.working = 1
} else {
sh.wl.working = 0
}
@ -517,9 +609,9 @@ func (sh *serviceDescription) authorize(w http.ResponseWriter, r *http.Request)
ServiceCode: sh.ServiceCode,
Platform: authtype,
Uid: uid,
//Token: accesstoken,
Sk: newsession,
Expired: expired,
Email: email,
Sk: newsession,
Expired: expired,
//RefreshToken: queryvals.Get("rt"),
}
@ -604,8 +696,6 @@ func (sh *serviceDescription) ServeHTTP(w http.ResponseWriter, r *http.Request)
} else {
// TODO : 세션키와 authtoken을 헤더로 받아서 accid 조회
queryvals := r.URL.Query()
//token := queryvals.Get("token")
token := "" // 더이상 쓰지 않는다.
sk := queryvals.Get("sk")
//if len(token) == 0 || len(sk) == 0 {
@ -617,14 +707,53 @@ func (sh *serviceDescription) ServeHTTP(w http.ResponseWriter, r *http.Request)
// TODO : 각 서버에 있는 자산? 캐릭터 정보를 보여줘야 하나. 뭘 보여줄지는 프로젝트에 문의
// 일단 서버 종류만 내려보내자
// 세션키가 있는지 확인
if _, ok := sh.auths.IsValid(sk, token); !ok {
logger.Println("sessionkey is not valid :", sk, token)
if _, ok := sh.auths.IsValid(sk, ""); !ok {
logger.Println("sessionkey is not valid :", sk)
w.WriteHeader(http.StatusBadRequest)
return
}
divstrptr := atomic.LoadPointer(&sh.divisionsSerialized)
divstr := *(*string)(divstrptr)
w.Write([]byte(divstr))
if divname := queryvals.Get("div"); len(divname) > 0 {
divname = strings.Trim(divname, `"`)
// 점검중인지 아닌지 확인
// 점검중이어도 입장이 가능한 인원이 있다.
div := sh.Divisions[divname]
if div != nil {
switch div.State {
case DivisionState_FullOpen:
w.Write([]byte(fmt.Sprintf(`{"service":"%s"}`, div.Url)))
case DivisionState_RestrictedOpen:
// 점검중인데 일부 권한을 갖고 있는 유저만 들어갈 수 있는 상태
cell := sh.auths.QuerySession(sk, "")
if cell == nil {
logger.Println("sessionkey is not valid :", sk)
w.WriteHeader(http.StatusBadRequest)
return
}
if sh.wl.hasAuth(cell.ToAuthinfo().Email, cell.ToAuthinfo().Platform, whitelistAuthType_QA) {
// qa 권한이면 입장 가능
w.Write([]byte(fmt.Sprintf(`{"service":"%s"}`, div.Url)))
} else if div.Maintenance != nil {
// 권한이 없으므로 공지
w.Write([]byte(fmt.Sprintf(`{"notice":"%s"}`, div.Maintenance.link)))
} else {
logger.Println("div.Maintenance is nil :", divname)
}
case DivisionState_Maintenance:
// 점검중. 아무도 못들어감
if div.Maintenance != nil {
w.Write([]byte(fmt.Sprintf(`{"notice":"%s"}`, div.Maintenance.link)))
} else {
logger.Println("div.Maintenance is nil :", divname)
}
}
}
} else {
divstrptr := atomic.LoadPointer(&sh.divisionsSerialized)
divstr := *(*string)(divstrptr)
w.Write([]byte(divstr))
}
}
}

68
core/watch.go
View File

@ -31,6 +31,10 @@ type servicePipelineDocument struct {
Service *serviceDescription `bson:"fullDocument"`
}
type filePipelineDocument struct {
File *fileDocumentDesc `bson:"fullDocument"`
}
type whilelistPipelineDocument struct {
OperationType string `bson:"operationType"`
DocumentKey struct {
@ -123,6 +127,70 @@ func (mg *Maingate) watchWhitelistCollection(parentctx context.Context) {
}
}
func (mg *Maingate) watchFileCollection(parentctx context.Context, serveMux *http.ServeMux, prefix string) {
defer func() {
s := recover()
if s != nil {
logger.Error(s)
}
}()
matchStage := bson.D{
{
Key: "$match", Value: bson.D{
{Key: "operationType", Value: bson.D{
{Key: "$in", Value: bson.A{
"insert",
}},
}},
},
}}
projectStage := bson.D{
{
Key: "$project", Value: bson.D{
{Key: "fullDocument", Value: 1},
},
},
}
var stream *mongo.ChangeStream
var err error
var ctx context.Context
for {
if stream == nil {
stream, err = mg.mongoClient.Watch(CollectionFile, mongo.Pipeline{matchStage, projectStage}, options.ChangeStream().SetFullDocument(options.UpdateLookup))
if err != nil {
logger.Error("watchFileCollection watch failed :", err)
time.Sleep(time.Second)
continue
}
ctx = context.TODO()
}
changed := stream.TryNext(ctx)
if ctx.Err() != nil {
logger.Error("watchFileCollection stream.TryNext failed. process should be restarted! :", ctx.Err().Error())
break
}
if !changed {
if stream.Err() != nil || stream.ID() == 0 {
logger.Error("watchServiceCollection stream error :", stream.Err())
stream.Close(ctx)
stream = nil
} else {
time.Sleep(time.Second)
}
continue
}
var data filePipelineDocument
if err := stream.Decode(&data); err == nil {
data.File.save()
}
}
}
func (mg *Maingate) watchServiceCollection(parentctx context.Context, serveMux *http.ServeMux, prefix string) {
defer func() {
s := recover()