ayo/houston
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

로그 append와 업로드를 분리

Browse Source
This commit is contained in:
mountain
2024-11-07 13:47:53 +09:00
parent 4d6665b64a
commit 2ddbae07b2
1 changed files with 31 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
client/operation.go
View File

@ -48,19 +48,14 @@ func lastExecutionArgs(verpath string) []string {
var errUploadZipLogFailed = errors.New("not ok")
func (hc *houstonClient) uploadRawLogFile(logFile string, name string, version string) error {
t, err := times.Stat(logFile)
if err != nil {
return err
}
func (hc *houstonClient) uploadLogFile(logFile string, name string, version string) error {
file, err := os.Open(logFile)
if err != nil {
return err
}
if file == nil {
return errors.New("uploadRawLogFile failed : " + logFile)
return errors.New("uploadLogFile failed : " + logFile)
}
defer file.Close()
@ -74,7 +69,7 @@ func (hc *houstonClient) uploadRawLogFile(logFile string, name string, version s
// createTime := file.
req.Header.Set("Houston-Service-Name", name)
req.Header.Set("Houston-Service-Version", version)
req.Header.Set("Houston-Service-Filename", t.BirthTime().UTC().Format(time.DateOnly)+"."+hn+path.Ext(logFile))
req.Header.Set("Houston-Service-Filename", path.Base(logFile)+"."+hn+path.Ext(logFile))
req.Header.Set("Content-Type", "application/zip")
resp, err := http.DefaultClient.Do(req)
if err != nil {
@ -89,25 +84,33 @@ func (hc *houstonClient) uploadRawLogFile(logFile string, name string, version s
return nil
}
func (hc *houstonClient) uploadZipLogFile(zipFile string, name string, version string) error {
zf, err := os.Open(zipFile)
func (hc *houstonClient) uploadToAppendLog(logFile string, name string, version string) error {
t, err := times.Stat(logFile)
if err != nil {
return err
}
if zf == nil {
return errUploadZipLogFailed
file, err := os.Open(logFile)
if err != nil {
return err
}
defer zf.Close()
if file == nil {
return errors.New("uploadRuploadLogFileawLogFile failed : " + logFile)
}
req, err := http.NewRequest("POST", hc.config.HttpAddress+"/upload", zf)
defer file.Close()
req, err := http.NewRequest("POST", hc.config.HttpAddress+"/upload", file)
if err != nil {
logger.Println(err)
}
hn, _ := os.Hostname()
// createTime := file.
req.Header.Set("Houston-Service-Name", name)
req.Header.Set("Houston-Service-Version", version)
req.Header.Set("Houston-Service-Filename", path.Base(filepath.ToSlash(zipFile)))
req.Header.Set("Houston-Service-Filename", t.BirthTime().UTC().Format(time.DateOnly)+"."+hn+path.Ext(logFile))
req.Header.Set("Content-Type", "application/zip")
resp, err := http.DefaultClient.Do(req)
if err != nil {
@ -385,15 +388,6 @@ func (hc *houstonClient) launch(meta *procmeta) error {
reader := bufio.NewReader(r)
readingMetric := false
defer func() {
if targetFile != nil {
targetFile.Close()
targetFile = nil
hc.uploadRawLogFile(currentFilePath, meta.name, meta.version)
}
metricExporter.Shutdown()
}()
var metricBuffer []byte
wipeLogFile := func() {
@ -401,10 +395,15 @@ func (hc *houstonClient) launch(meta *procmeta) error {
if targetFile != nil {
targetFile.Close()
targetFile = nil
hc.uploadRawLogFile(currentFilePath, meta.name, meta.version)
go hc.uploadToAppendLog(currentFilePath, meta.name, meta.version)
}
}
defer func() {
wipeLogFile()
metricExporter.Shutdown()
}()
currentTime := time.Now().UTC()
for {
now := time.Now().UTC()
@ -500,23 +499,19 @@ func (hc *houstonClient) launch(meta *procmeta) error {
}
reader := bufio.NewReader(r)
defer func() {
if targetFile != nil {
targetFile.Close()
targetFile = nil
}
hc.uploadProcFiles(meta, "logs/*"+ext, true)
}()
wipeLogFile := func() {
total = 0
if targetFile != nil {
targetFile.Close()
targetFile = nil
go hc.uploadRawLogFile(currentFilePath, meta.name, meta.version)
go hc.uploadToAppendLog(currentFilePath, meta.name, meta.version)
}
}
defer func() {
wipeLogFile()
}()
currentTime := time.Now().UTC()
for {
now := time.Now().UTC()
@ -728,7 +723,7 @@ func (hc *houstonClient) uploadProcFiles(child *procmeta, filter string, deleteA
_, matches := findMatchFiles(hc.config.StorageRoot, child.name, child.version, filter)
go func() {
for _, filename := range matches {
if err := hc.uploadRawLogFile(filename, child.name, child.version); err != nil {
if err := hc.uploadLogFile(filename, child.name, child.version); err != nil {
break
}
if deleteAfterUpload {
@ -752,7 +747,7 @@ func (hc *houstonClient) uploadFiles(req *shared.UploadRequest) error {
_, matches := findMatchFiles(hc.config.StorageRoot, req.Name, req.Version, req.Filter)
go func() {
for _, filename := range matches {
if err := hc.uploadRawLogFile(filename, req.Name, req.Version); err != nil {
if err := hc.uploadLogFile(filename, req.Name, req.Version); err != nil {
break
}
os.Remove(filename)