로그 파일을 압축하지 않고 하나씩 업로드
This commit is contained in:
@ -15,11 +15,14 @@ import (
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/djherbis/times"
|
||||
|
||||
"github.com/Knetic/govaluate"
|
||||
"repositories.action2quare.com/ayo/gocommon/logger"
|
||||
"repositories.action2quare.com/ayo/gocommon/metric"
|
||||
@ -45,6 +48,47 @@ func lastExecutionArgs(verpath string) []string {
|
||||
|
||||
var errUploadZipLogFailed = errors.New("not ok")
|
||||
|
||||
func (hc *houstonClient) uploadRawLogFile(logFile string, name string, version string) error {
|
||||
t, err := times.Stat(logFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := os.Open(logFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if file == nil {
|
||||
return errors.New("uploadRawLogFile failed : " + logFile)
|
||||
}
|
||||
|
||||
defer file.Close()
|
||||
|
||||
req, err := http.NewRequest("POST", hc.config.HttpAddress+"/upload", file)
|
||||
if err != nil {
|
||||
logger.Println(err)
|
||||
}
|
||||
|
||||
hn, _ := os.Hostname()
|
||||
// createTime := file.
|
||||
req.Header.Set("Houston-Service-Name", name)
|
||||
req.Header.Set("Houston-Service-Version", version)
|
||||
req.Header.Set("Houston-Service-Filename", t.BirthTime().UTC().Format(time.DateOnly)+"."+hn+path.Ext(logFile))
|
||||
req.Header.Set("Content-Type", "application/zip")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return errUploadZipLogFailed
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (hc *houstonClient) uploadZipLogFile(zipFile string, name string, version string) error {
|
||||
zf, err := os.Open(zipFile)
|
||||
if err != nil {
|
||||
@ -99,6 +143,7 @@ func findMatchFiles(storageRoot, name, version, filter string) (string, []string
|
||||
|
||||
out = append(out, file)
|
||||
}
|
||||
slices.Sort(out)
|
||||
return root, out
|
||||
}
|
||||
|
||||
@ -318,33 +363,59 @@ func (hc *houstonClient) launch(meta *procmeta) error {
|
||||
hc.siblingProcIndex[key] = runningFlags
|
||||
}()
|
||||
|
||||
metricExporter := metric.NewPrometheusExport(hc.config.MetricNamespace)
|
||||
|
||||
defer func() {
|
||||
reco := recover()
|
||||
if reco != nil {
|
||||
logger.Println(reco)
|
||||
}
|
||||
|
||||
r.Close()
|
||||
}()
|
||||
|
||||
metricExporter := metric.NewPrometheusExport(hc.config.MetricNamespace)
|
||||
defer metricExporter.Shutdown()
|
||||
|
||||
total := 0
|
||||
hn, _ := os.Hostname()
|
||||
var targetFile *os.File
|
||||
var currentFilePath string
|
||||
|
||||
ext := path.Ext(logfilePath)
|
||||
head := logfilePath[:len(logfilePath)-len(ext)]
|
||||
if len(head) > 0 && !strings.HasSuffix(head, "/") {
|
||||
head += "."
|
||||
}
|
||||
reader := bufio.NewReader(r)
|
||||
readingMetric := false
|
||||
ext = "." + hn + ext
|
||||
|
||||
defer func() {
|
||||
if targetFile != nil {
|
||||
targetFile.Close()
|
||||
targetFile = nil
|
||||
hc.uploadRawLogFile(currentFilePath, meta.name, meta.version)
|
||||
}
|
||||
metricExporter.Shutdown()
|
||||
}()
|
||||
|
||||
var metricBuffer []byte
|
||||
|
||||
wipeLogFile := func() {
|
||||
total = 0
|
||||
if targetFile != nil {
|
||||
targetFile.Close()
|
||||
targetFile = nil
|
||||
hc.uploadRawLogFile(currentFilePath, meta.name, meta.version)
|
||||
}
|
||||
}
|
||||
|
||||
currentTime := time.Now().UTC()
|
||||
for {
|
||||
now := time.Now().UTC()
|
||||
if now.YearDay() != currentTime.YearDay() {
|
||||
wipeLogFile()
|
||||
}
|
||||
|
||||
if targetFile == nil {
|
||||
currentFile := head + time.Now().UTC().Format(".2006-01-02.150405") + ext
|
||||
targetFile, _ = os.OpenFile(currentFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
|
||||
currentTime = time.Now().UTC()
|
||||
currentFilePath = head + currentTime.Format("2006-01-02.150405") + ext
|
||||
targetFile, _ = os.OpenFile(currentFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
|
||||
}
|
||||
|
||||
buff, err := reader.ReadBytes('\n')
|
||||
@ -402,12 +473,8 @@ func (hc *houstonClient) launch(meta *procmeta) error {
|
||||
}
|
||||
total += len(buff)
|
||||
|
||||
if total > 1024*1024 {
|
||||
total = 0
|
||||
targetFile.Close()
|
||||
targetFile = nil
|
||||
|
||||
hc.uploadProcFiles(meta, "logs/*"+ext, true)
|
||||
if total > 1024 {
|
||||
wipeLogFile()
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -423,18 +490,44 @@ func (hc *houstonClient) launch(meta *procmeta) error {
|
||||
defer r.Close()
|
||||
|
||||
total := 0
|
||||
hn, _ := os.Hostname()
|
||||
|
||||
var targetFile *os.File
|
||||
var currentFilePath string
|
||||
ext := path.Ext(logfilePath)
|
||||
head := logfilePath[:len(logfilePath)-len(ext)]
|
||||
if len(head) > 0 {
|
||||
head += "."
|
||||
}
|
||||
reader := bufio.NewReader(r)
|
||||
ext = "." + hn + ext
|
||||
|
||||
defer func() {
|
||||
if targetFile != nil {
|
||||
targetFile.Close()
|
||||
targetFile = nil
|
||||
}
|
||||
hc.uploadProcFiles(meta, "logs/*"+ext, true)
|
||||
}()
|
||||
|
||||
wipeLogFile := func() {
|
||||
total = 0
|
||||
if targetFile != nil {
|
||||
targetFile.Close()
|
||||
targetFile = nil
|
||||
hc.uploadRawLogFile(currentFilePath, meta.name, meta.version)
|
||||
}
|
||||
}
|
||||
|
||||
currentTime := time.Now().UTC()
|
||||
for {
|
||||
now := time.Now().UTC()
|
||||
if now.YearDay() != currentTime.YearDay() {
|
||||
wipeLogFile()
|
||||
}
|
||||
|
||||
if targetFile == nil {
|
||||
currentFile := head + time.Now().UTC().Format(".2006-01-02.150405") + ext
|
||||
targetFile, _ = os.OpenFile(currentFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
|
||||
currentTime = time.Now().UTC()
|
||||
currentFilePath = head + currentTime.Format("2006-01-02.150405") + ext
|
||||
targetFile, _ = os.OpenFile(currentFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
|
||||
}
|
||||
|
||||
buff, errRead := reader.ReadBytes('\n')
|
||||
@ -456,11 +549,7 @@ func (hc *houstonClient) launch(meta *procmeta) error {
|
||||
total += len(buff)
|
||||
|
||||
if total > 1024*1024 {
|
||||
total = 0
|
||||
targetFile.Close()
|
||||
targetFile = nil
|
||||
|
||||
hc.uploadProcFiles(meta, "logs/*"+ext, true)
|
||||
wipeLogFile()
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -517,7 +606,7 @@ func (hc *houstonClient) launch(meta *procmeta) error {
|
||||
if len(evalfile) > 0 {
|
||||
evalfile = path.Join(logfolder, evalfile)
|
||||
} else {
|
||||
evalfile = path.Join(logfolder, path.Base(meta.cmd.Args[0]))
|
||||
evalfile = logfolder + "/"
|
||||
}
|
||||
|
||||
go stdReader(meta.name, stdout, index, evalfile+".log")
|
||||
@ -636,27 +725,17 @@ func (hc *houstonClient) restartChildProcess(req *shared.RestartProcessRequest,
|
||||
|
||||
func (hc *houstonClient) uploadProcFiles(child *procmeta, filter string, deleteAfterUpload bool) {
|
||||
logger.Println("uploadFiles found :", child.version, child.name)
|
||||
root, matches := findMatchFiles(hc.config.StorageRoot, child.name, child.version, filter)
|
||||
|
||||
go func(deleteAfterUpload bool, root string, matches []string) {
|
||||
zipFile, err := zipCompressFiles(root, matches)
|
||||
if err == nil && len(zipFile) > 0 {
|
||||
if err = hc.uploadZipLogFile(zipFile, child.name, child.version); err == nil {
|
||||
if deleteAfterUpload {
|
||||
for _, fn := range matches {
|
||||
if len(fn) > 0 {
|
||||
os.Remove(fn)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logger.Println("uploadZipLogFile failed :", err)
|
||||
_, matches := findMatchFiles(hc.config.StorageRoot, child.name, child.version, filter)
|
||||
go func() {
|
||||
for _, filename := range matches {
|
||||
if err := hc.uploadRawLogFile(filename, child.name, child.version); err != nil {
|
||||
break
|
||||
}
|
||||
if deleteAfterUpload {
|
||||
os.Remove(filename)
|
||||
}
|
||||
os.Remove(zipFile)
|
||||
} else if err != nil {
|
||||
logger.Println("zipLogFiles failed :", err)
|
||||
}
|
||||
}(deleteAfterUpload, root, matches)
|
||||
}()
|
||||
}
|
||||
|
||||
func (hc *houstonClient) uploadFiles(req *shared.UploadRequest) error {
|
||||
@ -670,22 +749,15 @@ func (hc *houstonClient) uploadFiles(req *shared.UploadRequest) error {
|
||||
|
||||
// 실행 중이 아닌 폴더에서도 대상을 찾는다
|
||||
// 전체 파일을 대상으로
|
||||
root, matches := findMatchFiles(hc.config.StorageRoot, req.Name, req.Version, req.Filter)
|
||||
zipFile, err := zipCompressFiles(root, matches)
|
||||
if err == nil && len(zipFile) > 0 && len(zipFile) > 0 {
|
||||
if err = hc.uploadZipLogFile(zipFile, req.Name, req.Version); err == nil {
|
||||
for _, fn := range matches {
|
||||
if len(fn) > 0 {
|
||||
os.Remove(fn)
|
||||
}
|
||||
_, matches := findMatchFiles(hc.config.StorageRoot, req.Name, req.Version, req.Filter)
|
||||
go func() {
|
||||
for _, filename := range matches {
|
||||
if err := hc.uploadRawLogFile(filename, req.Name, req.Version); err != nil {
|
||||
break
|
||||
}
|
||||
os.Remove(zipFile)
|
||||
} else {
|
||||
logger.Println("uploadZipLogFile failed :", err)
|
||||
os.Remove(filename)
|
||||
}
|
||||
} else if err != nil {
|
||||
logger.Println("zipLogFiles failed :", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user