tavern 분리

This commit is contained in:
2023-05-24 16:10:00 +09:00
commit 1517f59fc3
13 changed files with 4225 additions and 0 deletions

1067
core/apiimpl.go Normal file

File diff suppressed because it is too large Load Diff

49
core/group.go Normal file
View File

@ -0,0 +1,49 @@
package core
import (
"net/url"
"repositories.action2quare.com/ayo/gocommon/wshandler"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type groupConfig struct {
UniqueIndex []string `json:"unique_index"`
SearchIndex []string `json:"search_index"`
MemberIndex []string `json:"member_index"`
TextSearchFields []string `json:"text_search_field"`
InviteExpire int32 `json:"invite_ttl"` // 그룹이 개인에게 보낸 초대장 만료 기한
CandidateExpire int32 `json:"candidate_ttl"` // 개인이 그룹에게 보낸 신청서 만료 기한
InviteeExlusive bool `json:"invitee_exlusive"`
InviteeIsMember bool `json:"invitee_is_member"`
MaxMember int `json:"max_member"`
Transient bool `json:"transient"`
Name string
}
type group interface {
Create(form url.Values, doc bson.M) (primitive.ObjectID, error)
Candidate(groupID primitive.ObjectID, memberID primitive.ObjectID, doc bson.M) error
Join(groupID primitive.ObjectID, memberID primitive.ObjectID, ticketID primitive.ObjectID, doc bson.M) (newTicketID primitive.ObjectID, err error)
FindTicketID(groupID primitive.ObjectID, memberID primitive.ObjectID) primitive.ObjectID
Invite(groupID primitive.ObjectID, memberID primitive.ObjectID, inviterDoc bson.M, inviteeDoc bson.M) (string, error)
UpdateGroupMember(groupID primitive.ObjectID, memberID primitive.ObjectID, ticketID primitive.ObjectID, doc bson.M) error
CancelInvitation(groupID primitive.ObjectID, ticketID primitive.ObjectID) error
AcceptInvitation(groupID primitive.ObjectID, mid primitive.ObjectID, ticketID primitive.ObjectID, member bson.M) (primitive.ObjectID, error)
DenyInvitation(groupID primitive.ObjectID, mid primitive.ObjectID, ticketID primitive.ObjectID) error
QueryInvitations(memberID primitive.ObjectID, after primitive.Timestamp) ([]bson.M, error)
Exist(groupID primitive.ObjectID, filter bson.M) (bool, error)
FindAll(filter bson.M, projection string, after primitive.Timestamp) ([]bson.M, error)
FindOne(groupID primitive.ObjectID, projection string) (bson.M, error)
QueryMembers(groupID primitive.ObjectID, requesterID primitive.ObjectID, projection string, after primitive.Timestamp) (map[string]bson.M, error)
QueryMember(groupID primitive.ObjectID, memberID primitive.ObjectID, ticketID primitive.ObjectID, projection string) (bson.M, error)
Leave(groupID primitive.ObjectID, memberID primitive.ObjectID, ticketID primitive.ObjectID) error
DropPausedMember(groupID primitive.ObjectID, memberID primitive.ObjectID) error
PauseMember(groupID primitive.ObjectID, memberID primitive.ObjectID, conn *wshandler.Richconn) error
UpdateMemberDocument(groupID primitive.ObjectID, memberID primitive.ObjectID, doc bson.M) error
Dismiss(groupID primitive.ObjectID) error
UpdateGroupDocument(groupID primitive.ObjectID, body []byte) error
}

1221
core/group_memory.go Normal file

File diff suppressed because it is too large Load Diff

1050
core/group_mongo.go Normal file

File diff suppressed because it is too large Load Diff

73
core/richconn.go Normal file
View File

@ -0,0 +1,73 @@
package core
import (
"context"
"repositories.action2quare.com/ayo/gocommon/logger"
"repositories.action2quare.com/ayo/gocommon/wshandler"
"github.com/go-redis/redis/v8"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type richConnOuter struct {
wsh *wshandler.WebsocketHandler
rc *wshandler.Richconn
}
func (sub richConnOuter) JoinTag(region string, tag primitive.ObjectID, tid primitive.ObjectID, hint string) error {
sub.wsh.JoinTag(region, tag, tid, sub.rc, hint)
wsh := sub.wsh
sub.rc.RegistOnCloseFunc(tag.Hex(), func() {
wsh.LeaveTag(region, tag, tid)
})
return nil
}
func (sub richConnOuter) LeaveTag(region string, tag primitive.ObjectID, tid primitive.ObjectID, hint string) error {
sub.SetStateInTag(region, tag, tid, "", hint)
return sub.wsh.LeaveTag(region, tag, tid)
}
func (sub richConnOuter) SetStateInTag(region string, tag primitive.ObjectID, tid primitive.ObjectID, state string, hint string) error {
return sub.wsh.SetStateInTag(region, tag, tid, state, hint)
}
func (sub richConnOuter) TurnGroupOnline(key string, gid primitive.ObjectID, score float64) error {
gidhex := gid.Hex()
_, err := sub.wsh.RedisSync.ZAdd(context.Background(), key, &redis.Z{Score: score, Member: gidhex}).Result()
if err != nil {
logger.Error("TurnGroupOnline failed. redis.ZAdd return err :", err)
return err
}
sub.rc.RegistOnCloseFunc(key, func() {
sub.wsh.RedisSync.ZRem(context.Background(), key, gidhex)
})
return nil
}
func (sub richConnOuter) TurnGroupOffline(key string, gid primitive.ObjectID) error {
f := sub.rc.UnregistOnCloseFunc(key)
if f != nil {
f()
} else {
sub.wsh.RedisSync.ZRem(context.Background(), key, gid.Hex())
}
return nil
}
func (sub richConnOuter) SendMessage(doc []byte) error {
return sub.rc.WriteBytes(doc)
}
func (sub richConnOuter) SendMessageToTag(region string, tag primitive.ObjectID, msg []byte) error {
sub.wsh.BroadcastRaw(region, tag, msg)
return nil
}
func (sub richConnOuter) CloseOnPurpose() error {
return sub.rc.Close()
}

195
core/rpc/connrpc.go Normal file
View File

@ -0,0 +1,195 @@
package rpc
import (
"bytes"
"encoding/gob"
"fmt"
"reflect"
"repositories.action2quare.com/ayo/gocommon/logger"
"repositories.action2quare.com/ayo/gocommon/wshandler"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
var Everybody = primitive.ObjectID([12]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
func init() {
gob.Register(bson.M{})
gob.Register(primitive.ObjectID{})
gob.Register(primitive.Timestamp{})
}
type RpcCaller struct {
publish func(bt []byte) error
}
func NewRpcCaller(f func(bt []byte) error) RpcCaller {
return RpcCaller{
publish: f,
}
}
type rpcCallContext struct {
alias primitive.ObjectID
publish func(bt []byte) error
}
func (c *RpcCaller) One(alias primitive.ObjectID) rpcCallContext {
return rpcCallContext{
alias: alias,
publish: c.publish,
}
}
func (c *RpcCaller) Everybody() rpcCallContext {
return rpcCallContext{
alias: Everybody,
publish: c.publish,
}
}
func IsCallerCalleeMethodMatch[Callee any]() error {
var caller rpcCallContext
var callee Callee
callerType := reflect.TypeOf(caller)
calleeType := reflect.TypeOf(callee)
for i := 0; i < callerType.NumMethod(); i++ {
callerMethod := callerType.Method(i)
calleeMethod, ok := calleeType.MethodByName(callerMethod.Name)
if !ok {
return fmt.Errorf("method '%s' of '%s' is missing", callerMethod.Name, calleeType.Name())
}
if calleeMethod.Func.Type().NumIn() != callerMethod.Func.Type().NumIn() {
return fmt.Errorf("method '%s' argument num is not match", callerMethod.Name)
}
if calleeMethod.Func.Type().NumOut() != callerMethod.Func.Type().NumOut() {
return fmt.Errorf("method '%s' out num is not match", callerMethod.Name)
}
for i := 1; i < calleeMethod.Func.Type().NumIn(); i++ {
if calleeMethod.Func.Type().In(i) != callerMethod.Func.Type().In(i) {
return fmt.Errorf("method '%s' argument is not match. %s-%s", callerMethod.Name, calleeMethod.Func.Type().In(i).Name(), callerMethod.Func.Type().In(i).Name())
}
}
}
return nil
}
type fnsig struct {
FunctionName string `bson:"fn"`
Args []any `bson:"args"`
}
func Encode[T any](prefix T, fn string, args ...any) ([]byte, error) {
m := append([]any{
prefix,
fn,
}, args...)
buff := new(bytes.Buffer)
encoder := gob.NewEncoder(buff)
err := encoder.Encode(m)
if err != nil {
logger.Error("rpcCallContext.send err :", err)
return nil, err
}
return buff.Bytes(), nil
}
func Decode[T any](src []byte) (*T, string, []any, error) {
var m []any
decoder := gob.NewDecoder(bytes.NewReader(src))
if err := decoder.Decode(&m); err != nil {
logger.Error("RpcCallee.Call err :", err)
return nil, "", nil, err
}
prfix := m[0].(T)
fn := m[1].(string)
return &prfix, fn, m[2:], nil
}
func decode(src []byte) (string, []any, error) {
var sig fnsig
decoder := gob.NewDecoder(bytes.NewReader(src))
if err := decoder.Decode(&sig); err != nil {
logger.Error("RpcCallee.Call err :", err)
return "", nil, err
}
return sig.FunctionName, sig.Args, nil
}
func (c *rpcCallContext) send(fn string, args ...any) error {
bt, err := Encode(c.alias, fn, args...)
if err != nil {
return err
}
return c.publish(bt)
}
type RpcCallee[T any] struct {
methods map[string]reflect.Method
create func(*wshandler.Richconn) *T
}
func NewRpcCallee[T any](createReceiverFunc func(*wshandler.Richconn) *T) RpcCallee[T] {
out := RpcCallee[T]{
methods: make(map[string]reflect.Method),
create: createReceiverFunc,
}
var tmp *T
tp := reflect.TypeOf(tmp)
for i := 0; i < tp.NumMethod(); i++ {
method := tp.Method(i)
out.methods[method.Name] = method
}
return out
}
func (r RpcCallee[T]) Call(rc *wshandler.Richconn, src []byte) error {
defer func() {
s := recover()
if s != nil {
logger.Error(s)
}
}()
fn, params, err := decode(src)
if err != nil {
logger.Error("RpcCallee.Call err :", err)
return err
}
method, ok := r.methods[fn]
if !ok {
err := fmt.Errorf("method '%s' is missing", fn)
logger.Error("RpcCallee.Call err :", err)
return err
}
receiver := r.create(rc)
args := []reflect.Value{
reflect.ValueOf(receiver),
}
for _, arg := range params {
args = append(args, reflect.ValueOf(arg))
}
rets := method.Func.Call(args)
if len(rets) > 0 && rets[len(rets)-1].Interface() != nil {
return rets[len(rets)-1].Interface().(error)
}
return nil
}

33
core/rpc/proxy.go Normal file
View File

@ -0,0 +1,33 @@
package rpc
import (
"go.mongodb.org/mongo-driver/bson/primitive"
)
func (c rpcCallContext) JoinTag(region string, tag primitive.ObjectID, tid primitive.ObjectID, hint string) error {
return c.send("JoinTag", region, tag, tid, hint)
}
func (c rpcCallContext) LeaveTag(region string, tag primitive.ObjectID, tid primitive.ObjectID, hint string) error {
return c.send("LeaveTag", region, tag, tid, hint)
}
func (c rpcCallContext) TurnGroupOnline(key string, gid primitive.ObjectID, score float64) error {
return c.send("TurnGroupOnline", key, gid, score)
}
func (c rpcCallContext) TurnGroupOffline(key string, gid primitive.ObjectID) error {
return c.send("TurnGroupOffline", key, gid)
}
func (c rpcCallContext) SetStateInTag(region string, tag primitive.ObjectID, tid primitive.ObjectID, state string, hint string) error {
return c.send("SetStateInTag", region, tag, tid, state, hint)
}
func (c rpcCallContext) SendMessage(doc []byte) error {
return c.send("SendMessage", doc)
}
func (c rpcCallContext) SendMessageToTag(region string, gid primitive.ObjectID, msg []byte) error {
return c.send("SendMessageToTag", region, gid, msg)
}
func (c rpcCallContext) CloseOnPurpose() error {
return c.send("CloseOnPurpose")
}

317
core/tavern.go Normal file
View File

@ -0,0 +1,317 @@
package core
import (
"context"
"errors"
"flag"
"io"
"net"
"net/http"
"reflect"
"strings"
common "repositories.action2quare.com/ayo/gocommon"
"repositories.action2quare.com/ayo/gocommon/logger"
"repositories.action2quare.com/ayo/gocommon/wshandler"
"repositories.action2quare.com/ayo/tavern/core/rpc"
"github.com/go-redis/redis/v8"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/bsonrw"
"go.mongodb.org/mongo-driver/bson/primitive"
)
const (
defaultMaxMemory = 32 << 10 // 32 KB
)
func writeBsonArr(w io.Writer, src []bson.M) error {
return writeBsonDoc(w, bson.M{
"r": src,
})
}
func onlineGroupQueryKey(prefix string) string {
return prefix + "_olg"
}
func writeBsonDoc[T any](w io.Writer, src T) error {
rw, err := bsonrw.NewBSONValueWriter(w)
if err != nil {
return err
}
enc, err := bson.NewEncoder(rw)
if err != nil {
return err
}
return enc.Encode(src)
}
func readBsonDoc(r io.Reader, src any) error {
body, err := io.ReadAll(r)
if err != nil {
return err
}
if len(body) == 0 {
return nil
}
decoder, err := bson.NewDecoder(bsonrw.NewBSONDocumentReader(body))
if err != nil {
return err
}
err = decoder.Decode(src)
if err != nil {
return err
}
return nil
}
type rpcCallDomain[T any] struct {
rpcCallChanName string
caller rpc.RpcCaller
callee rpc.RpcCallee[T]
methods map[string]reflect.Method
}
func createRpcCallDomain[CalleeType any](syncConn *redis.Client, creator func(*wshandler.Richconn) *CalleeType) rpcCallDomain[CalleeType] {
var tmp *CalleeType
methods := make(map[string]reflect.Method)
tp := reflect.TypeOf(tmp)
for i := 0; i < tp.NumMethod(); i++ {
method := tp.Method(i)
methods[method.Name] = method
}
rpcChanName := "conn_rpc_channel_" + tp.Name()
publishFunc := func(bt []byte) error {
_, err := syncConn.Publish(context.Background(), rpcChanName, bt).Result()
return err
}
return rpcCallDomain[CalleeType]{
rpcCallChanName: rpcChanName,
caller: rpc.NewRpcCaller(publishFunc),
callee: rpc.NewRpcCallee(creator),
methods: methods,
}
}
type TavernConfig struct {
common.RegionStorageConfig `json:",inline"`
GroupTypes map[string]*groupConfig `json:"tavern_group_types"`
MaingateApiToken string `json:"maingate_api_token"`
macAddr string
}
var config TavernConfig
type Tavern struct {
subTaverns []*subTavern
wsh *wshandler.WebsocketHandler
}
type subTavern struct {
mongoClient common.MongoClient
wsh *wshandler.WebsocketHandler
region string
groups map[string]group
methods map[string]reflect.Method
wshRpc rpcCallDomain[richConnOuter]
}
func getMacAddr() (string, error) {
ifas, err := net.Interfaces()
if err != nil {
return "", err
}
for _, ifa := range ifas {
a := ifa.HardwareAddr.String()
if a != "" {
a = strings.ReplaceAll(a, ":", "")
return a, nil
}
}
return "", errors.New("no net interface")
}
// New :
func New(context context.Context, wsh *wshandler.WebsocketHandler, inconfig *TavernConfig) (*Tavern, error) {
if !flag.Parsed() {
flag.Parse()
}
if inconfig == nil {
var loaded TavernConfig
if err := common.LoadConfig(&loaded); err != nil {
return nil, err
}
inconfig = &loaded
}
config = *inconfig
macaddr, err := getMacAddr()
if err != nil {
return nil, err
}
config.macAddr = macaddr
tv := Tavern{
wsh: wsh,
}
if err = tv.prepare(context); err != nil {
logger.Println("tavern prepare() failed :", err)
return nil, err
}
return &tv, nil
}
func (tv *Tavern) Destructor() {
tv.wsh.Destructor()
for _, st := range tv.subTaverns {
st.mongoClient.Close()
}
}
type groupPipelineDocument struct {
OperationType string `bson:"operationType"`
FullDocument map[string]any `bson:"fullDocument"`
DocumentKey struct {
Id primitive.ObjectID `bson:"_id"`
} `bson:"documentKey"`
UpdateDescription struct {
UpdatedFields bson.M `bson:"updatedFields"`
RemovedFileds bson.A `bson:"removedFields"`
TruncatedArrays bson.A `bson:"truncatedArrays"`
} `bson:"updateDescription"`
}
func (tv *Tavern) prepare(ctx context.Context) error {
for region, url := range config.RegionStorage {
var dbconn common.MongoClient
var err error
var groupinstance group
if err := rpc.IsCallerCalleeMethodMatch[richConnOuter](); err != nil {
return err
}
var tmp *subTavern
methods := make(map[string]reflect.Method)
tp := reflect.TypeOf(tmp)
for i := 0; i < tp.NumMethod(); i++ {
method := tp.Method(i)
methods[method.Name] = method
}
sub := &subTavern{
wsh: tv.wsh,
mongoClient: dbconn,
region: region,
methods: methods,
}
sub.wshRpc = createRpcCallDomain(tv.wsh.RedisSync, func(rc *wshandler.Richconn) *richConnOuter {
return &richConnOuter{wsh: sub.wsh, rc: rc}
})
groups := make(map[string]group)
for typename, cfg := range config.GroupTypes {
cfg.Name = typename
if cfg.Transient {
groupinstance, err = cfg.prepareInMemory(ctx, region, typename, tv.wsh)
} else {
if !dbconn.Connected() {
dbconn, err = common.NewMongoClient(ctx, url.Mongo, region)
if err != nil {
return err
}
}
groupinstance, err = cfg.preparePersistent(ctx, region, dbconn, tv.wsh)
}
if err != nil {
return err
}
groups[typename] = groupinstance
}
sub.groups = groups
tv.subTaverns = append(tv.subTaverns, sub)
}
return nil
}
func (tv *Tavern) RegisterHandlers(ctx context.Context, serveMux *http.ServeMux, prefix string) error {
// request는 항상 서비스 서버를 거쳐서 들어온다. [client] <--tls--> [service server] <--http--> tavern
// 클라이언트는 tavern으로부터 메시지를 수신할 뿐, 송신하지 못한다.
// 단, 요청은 https 서비스 서버를 통해 들어오고 클라이언트는 ws으로 수신만 한다는 원칙이 유지되어야 한다.(채팅 메시지는 예외?)
for _, sub := range tv.subTaverns {
var pattern string
if sub.region == "default" {
pattern = common.MakeHttpHandlerPattern(prefix, "api")
} else {
pattern = common.MakeHttpHandlerPattern(prefix, sub.region, "api")
}
serveMux.HandleFunc(pattern, sub.api)
deliveryChan := tv.wsh.DeliveryChannel(sub.region)
go sub.deliveryMessageHandler(deliveryChan)
}
return nil
}
func (sub *subTavern) api(w http.ResponseWriter, r *http.Request) {
defer func() {
s := recover()
if s != nil {
logger.Error(s)
}
io.Copy(io.Discard, r.Body)
r.Body.Close()
}()
// 서버에서 오는 요청만 처리
apitoken := r.Header.Get("MG-X-API-TOKEN")
if apitoken != config.MaingateApiToken {
// 서버가 보내는 쿼리만 허용
logger.Println("MG-X-API-TOKEN is missing")
w.WriteHeader(http.StatusBadRequest)
return
}
operation := r.URL.Query().Get("operation")
if len(operation) == 0 {
w.WriteHeader(http.StatusBadRequest)
return
}
method, ok := sub.methods[operation]
if !ok {
// 없는 operation
logger.Println("fail to call api. operation is not valid :", operation)
w.WriteHeader(http.StatusBadRequest)
return
}
if r.PostForm == nil {
r.ParseMultipartForm(defaultMaxMemory)
}
args := []reflect.Value{
reflect.ValueOf(sub),
reflect.ValueOf(w),
reflect.ValueOf(r),
}
method.Func.Call(args)
}