jinilkim/ds_new_user_analy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance logging and metrics for new user analysis

Browse Source 
- Updated logging setup to include a console formatter for concise output.
- Added detailed metrics for various dungeon modes (COOP, Solo, Survival, etc.) including entry counts, first results, escape counts, average survival times, and more.
- Improved retention analysis with additional checks for D2 to D6 retention.
- Enhanced progress logging during data collection and batch processing for better visibility.
- Updated result writing to include new metrics and adjusted headers accordingly.
- Refined retention funnel analysis to provide detailed insights into user retention rates across different days.
This commit is contained in:
Gnill82
2025-08-30 16:22:21 +09:00
parent 1f3a359abd
commit f431fdb19d
1 changed files with 517 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

641
ds_new_user_analy.py
View File

@ -82,7 +82,7 @@ logger = None
# ==============================================================================
def setup_logging(log_file_path: str) -> logging.Logger:
"""한국 시간 기준 로깅 설정"""
"""한국 시간 기준 로깅 설정 (콘솔 간결 모드)"""
class KSTFormatter(logging.Formatter):
def formatTime(self, record, datefmt=None):
@ -90,6 +90,40 @@ def setup_logging(log_file_path: str) -> logging.Logger:
ct = datetime.fromtimestamp(record.created, kst)
return ct.strftime('%Y-%m-%dT%H:%M:%S+09:00')
class ConsoleFormatter(logging.Formatter):
"""Console-optimized formatter - 간결한 콘솔 출력"""
def format(self, record):
# 콘솔에는 시간 없이 중요 메시지만 표시
if record.levelno >= logging.WARNING: # WARNING 이상은 상세 표시
return f"[{record.levelname}] {record.getMessage()}"
elif record.name == 'progress': # 진행률 추적용 특별 로거
return record.getMessage() # 시간 없이 출력
else:
# 일반 INFO 로그는 상당히 간결하게
msg = record.getMessage()
# 주요 정보만 콘솔에 표시
if any(keyword in msg for keyword in [
"던전 스토커즈 신규 유저 분석",
"분석 기간",
"세션 지표 포함",
"총 신규 유저",
"분석 완료",
"분석 요약",
"리텐션 퍼널",
"D0→D1", "D1→D2", "D2→D3", "D3→D4", "D4→D5", "D5→D6", "D6→D7",
"총 소요 시간",
"Retained_d",
"평균 활동 시간",
"="*20 # 구분선
]):
return msg
else:
return None
def formatMessage(self, record):
formatted = self.format(record)
return formatted if formatted is not None else ""
logger = logging.getLogger('NewUserAnalyzer')
logger.setLevel(logging.INFO)
@ -97,16 +131,18 @@ def setup_logging(log_file_path: str) -> logging.Logger:
for handler in logger.handlers[:]:
logger.removeHandler(handler)
# 파일 핸들러
# 파일 핸들러 (상세 로그)
file_handler = logging.FileHandler(log_file_path, encoding='utf-8')
file_formatter = KSTFormatter('[%(levelname)s] %(asctime)s - %(message)s')
file_handler.setFormatter(file_formatter)
logger.addHandler(file_handler)
# 콘솔 핸들러
# 콘솔 핸들러 (간결 로그)
console_handler = logging.StreamHandler()
console_formatter = KSTFormatter('[%(levelname)s] %(asctime)s - %(message)s')
console_formatter = ConsoleFormatter()
console_handler.setFormatter(console_formatter)
# 콘솔용 필터 추가
console_handler.addFilter(lambda record: console_formatter.format(record) is not None)
logger.addHandler(console_handler)
# 외부 라이브러리 로그 억제
@ -196,11 +232,37 @@ def get_fixed_metrics_config() -> Dict[str, Dict]:
"agg_type": "max_timestamp"
},
# ==================== 3.3 던전 플레이 성과 ====================
"dungeon_entry_count": {
# ==================== 3.3 던전 플레이 성과 (모드별) ====================
# 각 모드별 entry count
"COOP_entry_count": {
"index": "ds-logs-live-survival_sta",
"time_range": "d0",
"agg_type": "count"
"agg_type": "count",
"filters": [{"term": {"body.dungeon_mode.keyword": "COOP"}}]
},
"Solo_entry_count": {
"index": "ds-logs-live-survival_sta",
"time_range": "d0",
"agg_type": "count",
"filters": [{"term": {"body.dungeon_mode.keyword": "Solo"}}]
},
"Survival_entry_count": {
"index": "ds-logs-live-survival_sta",
"time_range": "d0",
"agg_type": "count",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival"}}]
},
"Survival_BOT_entry_count": {
"index": "ds-logs-live-survival_sta",
"time_range": "d0",
"agg_type": "count",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival_BOT"}}]
},
"Survival_Unprotected_entry_count": {
"index": "ds-logs-live-survival_sta",
"time_range": "d0",
"agg_type": "count",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival_Unprotected"}}]
},
"dungeon_first_mode": {
"index": "ds-logs-live-survival_sta",
@ -220,67 +282,292 @@ def get_fixed_metrics_config() -> Dict[str, Dict]:
"agg_type": "first_value",
"field": "body.result"
},
"dungeon_escape_count": {
# 각 모드별 첫 플레이 결과
"COOP_first_result": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "first_value",
"field": "body.result",
"filters": [{"term": {"body.dungeon_mode.keyword": "COOP"}}]
},
"Solo_first_result": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "first_value",
"field": "body.result",
"filters": [{"term": {"body.dungeon_mode.keyword": "Solo"}}]
},
"Survival_first_result": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "first_value",
"field": "body.result",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival"}}]
},
"Survival_BOT_first_result": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "first_value",
"field": "body.result",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival_BOT"}}]
},
"Survival_Unprotected_first_result": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "first_value",
"field": "body.result",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival_Unprotected"}}]
},
# 각 모드별 escape count
"COOP_escape_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "count",
"filters": [{"term": {"body.result": 1}}]
"filters": [{"term": {"body.result": 1}}, {"term": {"body.dungeon_mode.keyword": "COOP"}}]
},
"avg_survival_time": {
"Solo_escape_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "count",
"filters": [{"term": {"body.result": 1}}, {"term": {"body.dungeon_mode.keyword": "Solo"}}]
},
"Survival_escape_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "count",
"filters": [{"term": {"body.result": 1}}, {"term": {"body.dungeon_mode.keyword": "Survival"}}]
},
"Survival_BOT_escape_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "count",
"filters": [{"term": {"body.result": 1}}, {"term": {"body.dungeon_mode.keyword": "Survival_BOT"}}]
},
"Survival_Unprotected_escape_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "count",
"filters": [{"term": {"body.result": 1}}, {"term": {"body.dungeon_mode.keyword": "Survival_Unprotected"}}]
},
# 각 모드별 avg survival time
"COOP_avg_survival_time": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "avg",
"field": "body.play_stats.playtime"
"field": "body.play_stats.playtime",
"filters": [{"term": {"body.dungeon_mode.keyword": "COOP"}}]
},
"max_survival_time": {
"Solo_avg_survival_time": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "avg",
"field": "body.play_stats.playtime",
"filters": [{"term": {"body.dungeon_mode.keyword": "Solo"}}]
},
"Survival_avg_survival_time": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "avg",
"field": "body.play_stats.playtime",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival"}}]
},
"Survival_BOT_avg_survival_time": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "avg",
"field": "body.play_stats.playtime",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival_BOT"}}]
},
"Survival_Unprotected_avg_survival_time": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "avg",
"field": "body.play_stats.playtime",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival_Unprotected"}}]
},
# 각 모드별 max survival time
"COOP_max_survival_time": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "max",
"field": "body.play_stats.playtime"
"field": "body.play_stats.playtime",
"filters": [{"term": {"body.dungeon_mode.keyword": "COOP"}}]
},
"total_armor_break": {
"Solo_max_survival_time": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "max",
"field": "body.play_stats.playtime",
"filters": [{"term": {"body.dungeon_mode.keyword": "Solo"}}]
},
"Survival_max_survival_time": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "max",
"field": "body.play_stats.playtime",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival"}}]
},
"Survival_BOT_max_survival_time": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "max",
"field": "body.play_stats.playtime",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival_BOT"}}]
},
"Survival_Unprotected_max_survival_time": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "max",
"field": "body.play_stats.playtime",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival_Unprotected"}}]
},
# 각 모드별 raid play count
"COOP_raid_play_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "sum",
"field": "body.play_stats.armor_break_cnt"
"field": "body.play_stats.raid_play",
"filters": [{"term": {"body.dungeon_mode.keyword": "COOP"}}]
},
"raid_play_count": {
"Solo_raid_play_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "sum",
"field": "body.play_stats.raid_play"
"field": "body.play_stats.raid_play",
"filters": [{"term": {"body.dungeon_mode.keyword": "Solo"}}]
},
"escape_count": {
"index": "ds-logs-live-dead",
"Survival_raid_play_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "count",
"filters": [{"term": {"body.inter_type": 0}}]
"agg_type": "sum",
"field": "body.play_stats.raid_play",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival"}}]
},
"Survival_BOT_raid_play_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "sum",
"field": "body.play_stats.raid_play",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival_BOT"}}]
},
"Survival_Unprotected_raid_play_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "sum",
"field": "body.play_stats.raid_play",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival_Unprotected"}}]
},
# ==================== 3.4 전투 성과 ====================
"monster_kill_count": {
"index": "ds-logs-live-monster_kill",
# ==================== 3.4 전투 성과 (모드별) ====================
# 각 모드별 monster kill count
"COOP_monster_kill_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "count"
"agg_type": "sum",
"field": "body.play_stats.monster_kill_cnt",
"filters": [{"term": {"body.dungeon_mode.keyword": "COOP"}}]
},
"player_kill_count": {
"index": "ds-logs-live-player_kill",
"Solo_monster_kill_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "count",
"target_field": "uid.keyword"
"agg_type": "sum",
"field": "body.play_stats.monster_kill_cnt",
"filters": [{"term": {"body.dungeon_mode.keyword": "Solo"}}]
},
"player_killed_count": {
"Survival_monster_kill_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "sum",
"field": "body.play_stats.monster_kill_cnt",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival"}}]
},
"Survival_BOT_monster_kill_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "sum",
"field": "body.play_stats.monster_kill_cnt",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival_BOT"}}]
},
"Survival_Unprotected_monster_kill_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "sum",
"field": "body.play_stats.monster_kill_cnt",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival_Unprotected"}}]
},
# 각 모드별 player kill count
"COOP_player_kill_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "sum",
"field": "body.play_stats.player_kill_cnt",
"filters": [{"term": {"body.dungeon_mode.keyword": "COOP"}}]
},
"Solo_player_kill_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "sum",
"field": "body.play_stats.player_kill_cnt",
"filters": [{"term": {"body.dungeon_mode.keyword": "Solo"}}]
},
"Survival_player_kill_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "sum",
"field": "body.play_stats.player_kill_cnt",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival"}}]
},
"Survival_BOT_player_kill_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "sum",
"field": "body.play_stats.player_kill_cnt",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival_BOT"}}]
},
"Survival_Unprotected_player_kill_count": {
"index": "ds-logs-live-survival_end",
"time_range": "d0",
"agg_type": "sum",
"field": "body.play_stats.player_kill_cnt",
"filters": [{"term": {"body.dungeon_mode.keyword": "Survival_Unprotected"}}]
},
# ==================== 3.4.2 사망 원인 분석 ====================
"death_PK": {
"index": "ds-logs-live-player_kill",
"time_range": "d0",
"agg_type": "count",
"target_field": "body.target_uid.keyword",
"use_body_target": True
},
"death_count": {
"death_GiveUp": {
"index": "ds-logs-live-dead",
"time_range": "d0",
"agg_type": "count",
"filters": [{"bool": {"must_not": {"term": {"body.inter_type": 0}}}}]
"filters": [{"term": {"body.inter_type": 0}}]
},
"death_Mob": {
"index": "ds-logs-live-dead",
"time_range": "d0",
"agg_type": "count",
"filters": [{"term": {"body.inter_type": 1}}]
},
"death_Trap": {
"index": "ds-logs-live-dead",
"time_range": "d0",
"agg_type": "count",
"filters": [{"term": {"body.inter_type": 10}}]
},
"death_Red": {
"index": "ds-logs-live-dead",
"time_range": "d0",
"agg_type": "count",
"filters": [{"term": {"body.inter_type": 11}}]
},
"death_Others": {
"index": "ds-logs-live-dead",
"time_range": "d0",
"agg_type": "count",
"filters": [{"bool": {"must_not": [{"terms": {"body.inter_type": [0, 1, 10, 11]}}]}}]
},
# ==================== 3.5 진행도 및 성장 ====================
@ -318,18 +605,8 @@ def get_fixed_metrics_config() -> Dict[str, Dict]:
"agg_type": "max",
"field": "body.guide_step"
},
"skill_points_earned": {
"index": "ds-logs-live-skill_point_get",
"time_range": "d0",
"agg_type": "count"
},
# ==================== 3.6 아이템 및 경제 ====================
"items_obtained_count": {
"index": "ds-logs-live-item_get",
"time_range": "d0",
"agg_type": "count"
},
"highest_item_grade": {
"index": "ds-logs-live-item_get",
"time_range": "d0",
@ -516,9 +793,29 @@ def get_fixed_metrics_config() -> Dict[str, Dict]:
"time_range": "d1",
"agg_type": "exists"
},
"retention_check_d2_6": {
"retention_check_d2": {
"index": "ds-logs-live-heartbeat",
"time_range": "d2_6",
"time_range": "d2",
"agg_type": "exists"
},
"retention_check_d3": {
"index": "ds-logs-live-heartbeat",
"time_range": "d3",
"agg_type": "exists"
},
"retention_check_d4": {
"index": "ds-logs-live-heartbeat",
"time_range": "d4",
"agg_type": "exists"
},
"retention_check_d5": {
"index": "ds-logs-live-heartbeat",
"time_range": "d5",
"agg_type": "exists"
},
"retention_check_d6": {
"index": "ds-logs-live-heartbeat",
"time_range": "d6",
"agg_type": "exists"
},
"retention_check_d7_plus": {
@ -582,6 +879,8 @@ def get_new_user_cohort_optimized(
# 신규 생성된 유저 수집
new_user_map = {} # uid -> {'create_time': ...}
page_count = 0
total_pages_estimate = 0
progress_logger = logging.getLogger('progress')
while True:
page_count += 1
@ -590,6 +889,12 @@ def get_new_user_cohort_optimized(
query["aggs"]["new_users"]["composite"]["after"] = after_key
try:
# 진행률 표시 (제자리 갱신)
if page_count == 1:
progress_logger.info("\r코호트 선정: 페이지 1 처리 중...")
else:
progress_logger.info(f"\r코호트 선정: 페이지 {page_count} 처리 중... (수집: {len(new_user_map)}명)")
logger.info(f"create_uid 페이지 {page_count} 처리 중...")
response = exponential_backoff_retry(
client.search,
@ -614,6 +919,8 @@ def get_new_user_cohort_optimized(
}
logger.info(f"create_uid 페이지 {page_count}: {len(buckets)}개 처리됨")
# 콘솔에는 간결하게
progress_logger.info(f"\r코호트 선정: 페이지 {page_count} 완료 (수집: {len(new_user_map)}명)")
after_key = response["aggregations"]["new_users"].get("after_key")
if not after_key:
@ -623,6 +930,8 @@ def get_new_user_cohort_optimized(
logger.error(f"create_uid 처리 중 오류 (페이지 {page_count}): {e}")
break
# 최종 진행률 표시
progress_logger.info(f"\r코호트 선정 완료: 총 {len(new_user_map)}명 확인 \n")
logger.info(f"총 {len(new_user_map)}명의 신규 유저 확인됨")
# Step 2: 모든 create_uid 유저를 cohort에 추가
@ -648,6 +957,7 @@ def get_new_user_cohort_optimized(
}
total_users += 1
progress_logger.info(f"\r코호트 초기화 완료: {total_users}명 ")
logger.info(f"cohort에 {total_users}명의 신규 유저 추가 완료")
# Step 3: login_comp 인덱스에서 추가 정보 수집 (auth.id 1순위)
@ -728,6 +1038,7 @@ def get_new_user_cohort_optimized(
except Exception as e:
logger.error(f"login_comp 정보 수집 중 오류: {e}")
progress_logger.info(f"\rlogin_comp 정보 수집: {len(login_comp_collected)}/{total_users}명 완료 ")
logger.info(f"login_comp에서 {len(login_comp_collected)}명의 정보 수집 완료")
# Step 4: log_return_to_lobby 인덱스에서 차선 정보 수집 (auth.id 2순위)
@ -806,6 +1117,7 @@ def get_new_user_cohort_optimized(
except Exception as e:
logger.error(f"log_return_to_lobby 정보 수집 중 오류: {e}")
progress_logger.info(f"\rlobby 추가 정보 수집: {len(lobby_collected)}명 완료 ")
logger.info(f"log_return_to_lobby에서 {len(lobby_collected)}명의 차선 정보 수집 완료")
# 최종 통계
@ -840,7 +1152,15 @@ def build_fixed_msearch_queries(
d1_start = d0_end
d1_end = (create_time_dt + timedelta(hours=48)).strftime('%Y-%m-%dT%H:%M:%SZ')
d2_start = d1_end
d6_end = (create_time_dt + timedelta(hours=168)).strftime('%Y-%m-%dT%H:%M:%SZ') # 168시간 = 7일
d2_end = (create_time_dt + timedelta(hours=72)).strftime('%Y-%m-%dT%H:%M:%SZ')
d3_start = d2_end
d3_end = (create_time_dt + timedelta(hours=96)).strftime('%Y-%m-%dT%H:%M:%SZ')
d4_start = d3_end
d4_end = (create_time_dt + timedelta(hours=120)).strftime('%Y-%m-%dT%H:%M:%SZ')
d5_start = d4_end
d5_end = (create_time_dt + timedelta(hours=144)).strftime('%Y-%m-%dT%H:%M:%SZ')
d6_start = d5_end
d6_end = (create_time_dt + timedelta(hours=168)).strftime('%Y-%m-%dT%H:%M:%SZ')
d7_plus_start = d6_end
for metric_name, config in metrics_config.items():
@ -849,8 +1169,16 @@ def build_fixed_msearch_queries(
time_filter = {"range": {"@timestamp": {"gte": d0_start, "lt": d0_end}}}
elif config["time_range"] == "d1":
time_filter = {"range": {"@timestamp": {"gte": d1_start, "lt": d1_end}}}
elif config["time_range"] == "d2_6":
time_filter = {"range": {"@timestamp": {"gte": d2_start, "lt": d6_end}}}
elif config["time_range"] == "d2":
time_filter = {"range": {"@timestamp": {"gte": d2_start, "lt": d2_end}}}
elif config["time_range"] == "d3":
time_filter = {"range": {"@timestamp": {"gte": d3_start, "lt": d3_end}}}
elif config["time_range"] == "d4":
time_filter = {"range": {"@timestamp": {"gte": d4_start, "lt": d4_end}}}
elif config["time_range"] == "d5":
time_filter = {"range": {"@timestamp": {"gte": d5_start, "lt": d5_end}}}
elif config["time_range"] == "d6":
time_filter = {"range": {"@timestamp": {"gte": d6_start, "lt": d6_end}}}
elif config["time_range"] == "d7_plus":
time_filter = {"range": {"@timestamp": {"gte": d7_plus_start}}}
else: # 기본값
@ -1066,7 +1394,8 @@ def process_fixed_batch(
) -> List[Dict]:
"""수정된 배치 처리 함수"""
logger.info(f"배치 처리 시작: {len(batch_uids)}명")
# 진행률 추적용 로거
progress_logger = logging.getLogger('progress')
try:
# 1. 세션 지표 계산 (--full 옵션일 때만)
@ -1086,6 +1415,8 @@ def process_fixed_batch(
msearch_queries = build_fixed_msearch_queries(batch_uids, cohort, metrics_config)
body_ndjson = "\n".join(msearch_queries) + "\n"
# 콘솔에는 간단히, 파일에는 상세히
progress_logger.info(f"\r배치 처리 중: {len(batch_uids)}명, {len(msearch_queries)//2}개 쿼리")
logger.info(f"msearch 실행: {len(msearch_queries)//2}개 쿼리")
msearch_responses = exponential_backoff_retry(
@ -1111,7 +1442,6 @@ def process_fixed_batch(
'nickname': user_data['nickname'],
'create_time': user_data.get('create_time_kst', 'N/A'),
'retention_status': 'Retained_d0', # 기본값, 나중에 업데이트
'last_active_day': 0, # 마지막 접속일 추가
'language': user_data['language'],
'country': user_data.get('country', 'N/A'),
'device': user_data['device'],
@ -1234,77 +1564,55 @@ def process_fixed_batch(
result[metric_name] = 0
# 모든 메트릭 처리 후 리텐션 상태 판정
# D7+ > D2~6 > D1 > D0 순서로 판정 (마지막 접속일 기준)
# D7+ > D6 > D5 > D4 > D3 > D2 > D1 > D0 순서로 판정 (마지막 접속일 기준)
if result.get('retention_check_d7_plus', 0) > 0:
result['retention_status'] = 'Retained_d7+'
result['last_active_day'] = 7 # 7+ 표시
elif result.get('retention_check_d2_6', 0) > 0:
result['retention_status'] = 'Retained_d2~6'
result['last_active_day'] = 2 # 정확한 날짜는 알 수 없으므로 2로 표시
elif result.get('retention_check_d6', 0) > 0:
result['retention_status'] = 'Retained_d6'
elif result.get('retention_check_d5', 0) > 0:
result['retention_status'] = 'Retained_d5'
elif result.get('retention_check_d4', 0) > 0:
result['retention_status'] = 'Retained_d4'
elif result.get('retention_check_d3', 0) > 0:
result['retention_status'] = 'Retained_d3'
elif result.get('retention_check_d2', 0) > 0:
result['retention_status'] = 'Retained_d2'
elif result.get('retention_check_d1', 0) > 0:
result['retention_status'] = 'Retained_d1'
result['last_active_day'] = 1
else:
# D0에만 접속 (기본값 유지)
result['retention_status'] = 'Retained_d0'
result['last_active_day'] = 0
# 계산된 지표
if result.get('dungeon_entry_count', 0) > 0:
escape_count = result.get('dungeon_escape_count', 0)
result['dungeon_escape_rate'] = round((escape_count / result['dungeon_entry_count']) * 100, 2)
# 모드별 계산된 지표
modes = ['COOP', 'Solo', 'Survival', 'Survival_BOT', 'Survival_Unprotected']
for mode in modes:
# 각 모드별 escape rate 계산
entry_key = f'{mode}_entry_count'
escape_key = f'{mode}_escape_count'
# 게임당 평균 데미지 계산을 위해 직접 쿼리 (제거된 필드들 대신)
dungeon_count = result['dungeon_entry_count']
try:
# 시간 범위 가져오기 (create_time 기준)
create_time_dt = user_data['create_time_dt']
d0_start = user_data['create_time_utc']
d0_end = (create_time_dt + timedelta(hours=24)).strftime('%Y-%m-%dT%H:%M:%SZ')
if result.get(entry_key, 0) > 0:
escape_count = result.get(escape_key, 0)
result[f'{mode}_escape_rate'] = round((escape_count / result[entry_key]), 4) # 소수점 4자리
# survival_end 인덱스에서 직접 데미지 합계 조회
damage_query = {
"size": 0,
"query": {
"bool": {
"filter": [
{"term": {"uid.keyword": uid}},
{"range": {"@timestamp": {"gte": d0_start, "lt": d0_end}}}
]
}
},
"aggs": {
"total_monster_damage": {"sum": {"field": "body.play_stats.damage_dealt_monster"}},
"total_player_damage": {"sum": {"field": "body.play_stats.damage_dealt_player"}}
}
}
# 각 모드별 평균 킬 수 계산
monster_kills = result.get(f'{mode}_monster_kill_count', 0)
player_kills = result.get(f'{mode}_player_kill_count', 0)
entry_count = result[entry_key]
damage_response = exponential_backoff_retry(
client.search,
index="ds-logs-live-survival_end",
body=damage_query,
request_timeout=DEFAULT_TIMEOUT
)
monster_damage = damage_response.get('aggregations', {}).get('total_monster_damage', {}).get('value', 0) or 0
player_damage = damage_response.get('aggregations', {}).get('total_player_damage', {}).get('value', 0) or 0
result['avg_damage_per_game_monster'] = round(monster_damage / dungeon_count, 2)
result['avg_damage_per_game_player'] = round(player_damage / dungeon_count, 2)
except Exception as e:
logger.warning(f"평균 데미지 계산 실패 (UID: {uid}): {e}")
result['avg_damage_per_game_monster'] = 0
result['avg_damage_per_game_player'] = 0
else:
result['dungeon_escape_rate'] = 0
result['avg_damage_per_game_monster'] = 0
result['avg_damage_per_game_player'] = 0
result[f'{mode}_avg_monster_kills'] = round(monster_kills / entry_count, 2)
result[f'{mode}_avg_player_kills'] = round(player_kills / entry_count, 2)
else:
result[f'{mode}_escape_rate'] = 0.0
result[f'{mode}_avg_monster_kills'] = 0.0
result[f'{mode}_avg_player_kills'] = 0.0
batch_results.append(result)
except Exception as e:
logger.warning(f"UID '{uid}' 처리 중 오류: {e}")
progress_logger.info(f"\r배치 완료: {len(batch_results)}명 ")
logger.info(f"배치 처리 완료: {len(batch_results)}명 성공")
return batch_results
@ -1334,6 +1642,9 @@ def process_cohort_fixed_parallel(
logger.info(f"분석 지표: {len(metrics_config)}개")
logger.info(f"세션 지표 포함: {'예' if include_session_metrics else '아니오'}")
# 진행률 추적용 로거
progress_logger = logging.getLogger('progress')
uid_list = list(cohort.keys())
chunks = [uid_list[i:i + batch_size] for i in range(0, len(uid_list), batch_size)]
@ -1346,6 +1657,7 @@ def process_cohort_fixed_parallel(
for chunk in chunks
}
completed_chunks = 0
with tqdm(total=len(chunks), desc="배치 처리 진행률") as pbar:
for future in as_completed(future_to_chunk):
chunk = future_to_chunk[future]
@ -1355,28 +1667,39 @@ def process_cohort_fixed_parallel(
all_results.extend(batch_results)
else:
failed_chunks.append(chunk)
completed_chunks += 1
# 콘솔 진행률 업데이트
progress_logger.info(f"\r병렬 처리 중: {completed_chunks}/{len(chunks)} 배치 완료, {len(all_results)}명 처리됨")
except Exception as e:
logger.warning(f"배치 처리 실패: {e}")
failed_chunks.append(chunk)
completed_chunks += 1
progress_logger.info(f"\r병렬 처리 중: {completed_chunks}/{len(chunks)} 배치 완료 (오류 1개), {len(all_results)}명 처리됨")
finally:
pbar.update(1)
# 실패한 청크 재처리
if failed_chunks:
logger.info(f"실패한 {len(failed_chunks)}개 배치 재처리 중...")
for chunk in failed_chunks:
progress_logger.info(f"\r재처리 중: {len(failed_chunks)}개 배치...")
for i, chunk in enumerate(failed_chunks):
try:
batch_results = process_fixed_batch(client, chunk, cohort, metrics_config, include_session_metrics)
all_results.extend(batch_results)
progress_logger.info(f"\r재처리 중: {i+1}/{len(failed_chunks)} 완료")
except Exception as e:
logger.error(f"재처리 실패: {e}")
# 최종 결과
progress_logger.info(f"\r병렬 처리 완료: 총 {len(all_results)}명 성공 \\n")
logger.info(f"2단계 완료: {len(all_results)}명 처리 성공")
logger.info("=" * 80)
return all_results
def write_fixed_results(results: List[Dict], output_path: str) -> None:
def write_fixed_results(results: List[Dict], output_path: str, include_session_metrics: bool = False) -> None:
"""수정된 결과 저장 (retention_d1 제거)"""
logger.info("=" * 80)
@ -1387,17 +1710,38 @@ def write_fixed_results(results: List[Dict], output_path: str) -> None:
return
# 수정된 헤더 (first_login_time 제거, create_time만 사용, country 및 last_active_day 추가)
# 기본 헤더
headers = [
'uid', 'auth_id', 'nickname', 'create_time', 'retention_status', 'last_active_day', 'language', 'country', 'device',
'active_seconds', 'total_playtime_minutes', 'session_count', 'avg_session_length', 'logout_abnormal',
'dungeon_entry_count', 'dungeon_first_mode', 'dungeon_first_stalker', 'dungeon_first_result',
'dungeon_escape_count', 'dungeon_escape_rate', 'avg_survival_time', 'max_survival_time',
'total_armor_break', 'raid_play_count', 'escape_count',
'monster_kill_count', 'player_kill_count', 'player_killed_count', 'death_count',
'avg_damage_per_game_monster', 'avg_damage_per_game_player',
'uid', 'auth_id', 'nickname', 'create_time', 'retention_status', 'language', 'country', 'device',
'dungeon_first_mode', 'dungeon_first_stalker', 'dungeon_first_result',
'death_PK', 'death_GiveUp', 'death_Mob', 'death_Trap', 'death_Red', 'death_Others'
]
# 세션 지표 (옵션에 따른 조건부 포함)
if include_session_metrics:
session_headers = ['active_seconds', 'total_playtime_minutes', 'session_count', 'avg_session_length', 'logout_abnormal']
headers = headers[:4] + session_headers + headers[4:] # retention_status 뒤에 삽입
# 각 모드별 헤더 추가 (삭제된 지표 제외)
modes = ['COOP', 'Solo', 'Survival', 'Survival_BOT', 'Survival_Unprotected']
for mode in modes:
headers.extend([
f'{mode}_entry_count',
f'{mode}_escape_count',
f'{mode}_escape_rate',
f'{mode}_avg_survival_time',
f'{mode}_max_survival_time',
f'{mode}_raid_play_count',
f'{mode}_first_result',
f'{mode}_avg_monster_kills',
f'{mode}_avg_player_kills'
])
# 나머지 헤더 (삭제된 지표 제외)
headers.extend([
'level_max', 'level_max_stalker', 'tutorial_entry', 'tutorial_completed',
'guide_quest_stage', 'skill_points_earned',
'items_obtained_count', 'highest_item_grade', 'blueprint_use_count', 'shop_buy_count',
'guide_quest_stage',
'highest_item_grade', 'blueprint_use_count', 'shop_buy_count',
'shop_sell_count', 'gold_spent', 'gold_earned', 'storage_in_count', 'storage_out_count',
'enchant_count', 'enchant_gold_spent',
'ingame_equip_count', 'object_interaction_count',
@ -1406,7 +1750,7 @@ def write_fixed_results(results: List[Dict], output_path: str) -> None:
'exchange_register_count', 'exchange_use_count', 'coupon_used',
'button_click_count', 'hideout_upgrade_count', 'hideout_max_level', 'season_pass_buy', 'season_pass_max_step',
'last_logout_time'
]
])
try:
with open(output_path, 'w', newline='', encoding='utf-8-sig') as csvfile:
@ -1499,24 +1843,73 @@ def main():
client, cohort, args.batch_size, args.max_workers, metrics_config, args.full
)
write_fixed_results(results, str(csv_file_path))
write_fixed_results(results, str(csv_file_path), args.full)
# 요약
if results:
total_users = len(results)
retained_d0 = sum(1 for r in results if r.get('retention_status') == 'Retained_d0')
retained_d1 = sum(1 for r in results if r.get('retention_status') == 'Retained_d1')
retained_d2_6 = sum(1 for r in results if r.get('retention_status') == 'Retained_d2~6')
retained_d7_plus = sum(1 for r in results if r.get('retention_status') == 'Retained_d7+')
retention_counts = {}
retention_groups = ['Retained_d0', 'Retained_d1', 'Retained_d2', 'Retained_d3', 'Retained_d4', 'Retained_d5', 'Retained_d6', 'Retained_d7+']
# 각 그룹별 카운트
for group in retention_groups:
retention_counts[group] = sum(1 for r in results if r.get('retention_status') == group)
logger.info("=" * 80)
logger.info("분석 요약")
logger.info("=" * 80)
logger.info(f"총 신규 유저: {total_users:,}명")
logger.info(f"D+0 리텐션: {retained_d0:,}명 ({(retained_d0/total_users)*100:.1f}%)")
logger.info(f"D+1 리텐션: {retained_d1:,}명 ({(retained_d1/total_users)*100:.1f}%)")
logger.info(f"D+2~6 리텐션: {retained_d2_6:,}명 ({(retained_d2_6/total_users)*100:.1f}%)")
logger.info(f"D+7+ 리텐션: {retained_d7_plus:,}명 ({(retained_d7_plus/total_users)*100:.1f}%)")
# 기본 리텐션 분포
for group in retention_groups:
count = retention_counts[group]
percentage = (count / total_users) * 100
logger.info(f"{group}: {count:,}명 ({percentage:.1f}%)")
# 리텐션 퍼널 요약 (단계별 + 누적)
logger.info("")
logger.info("=" * 50)
logger.info("리텐션 퍼널 분석")
logger.info("=" * 50)
# 누적 리텐션 계산 (D1 이상 접속한 유저들)
cumulative_d1_plus = sum(retention_counts[group] for group in retention_groups[1:]) # D1부터
cumulative_d2_plus = sum(retention_counts[group] for group in retention_groups[2:]) # D2부터
cumulative_d3_plus = sum(retention_counts[group] for group in retention_groups[3:]) # D3부터
cumulative_d4_plus = sum(retention_counts[group] for group in retention_groups[4:]) # D4부터
cumulative_d5_plus = sum(retention_counts[group] for group in retention_groups[5:]) # D5부터
cumulative_d6_plus = sum(retention_counts[group] for group in retention_groups[6:]) # D6부터
cumulative_d7_plus = retention_counts['Retained_d7+']
# 퍼널 분석 (단계별 잔존율 + 누적 리텐션율)
if cumulative_d1_plus > 0:
logger.info(f"D0→D1: {(cumulative_d1_plus/total_users)*100:.1f}% ({cumulative_d1_plus:,}/{total_users:,}명) | 누적: {(cumulative_d1_plus/total_users)*100:.1f}%")
if cumulative_d2_plus > 0:
d1_to_d2_rate = (cumulative_d2_plus/cumulative_d1_plus)*100
logger.info(f"D1→D2: {d1_to_d2_rate:.1f}% ({cumulative_d2_plus:,}/{cumulative_d1_plus:,}명) | 누적: {(cumulative_d2_plus/total_users)*100:.1f}%")
if cumulative_d3_plus > 0:
d2_to_d3_rate = (cumulative_d3_plus/cumulative_d2_plus)*100
logger.info(f"D2→D3: {d2_to_d3_rate:.1f}% ({cumulative_d3_plus:,}/{cumulative_d2_plus:,}명) | 누적: {(cumulative_d3_plus/total_users)*100:.1f}%")
if cumulative_d4_plus > 0:
d3_to_d4_rate = (cumulative_d4_plus/cumulative_d3_plus)*100
logger.info(f"D3→D4: {d3_to_d4_rate:.1f}% ({cumulative_d4_plus:,}/{cumulative_d3_plus:,}명) | 누적: {(cumulative_d4_plus/total_users)*100:.1f}%")
if cumulative_d5_plus > 0:
d4_to_d5_rate = (cumulative_d5_plus/cumulative_d4_plus)*100
logger.info(f"D4→D5: {d4_to_d5_rate:.1f}% ({cumulative_d5_plus:,}/{cumulative_d4_plus:,}명) | 누적: {(cumulative_d5_plus/total_users)*100:.1f}%")
if cumulative_d6_plus > 0:
d5_to_d6_rate = (cumulative_d6_plus/cumulative_d5_plus)*100
logger.info(f"D5→D6: {d5_to_d6_rate:.1f}% ({cumulative_d6_plus:,}/{cumulative_d5_plus:,}명) | 누적: {(cumulative_d6_plus/total_users)*100:.1f}%")
if cumulative_d7_plus > 0:
d6_to_d7_rate = (cumulative_d7_plus/cumulative_d6_plus)*100
logger.info(f"D6→D7+: {d6_to_d7_rate:.1f}% ({cumulative_d7_plus:,}/{cumulative_d6_plus:,}명) | 누적: {(cumulative_d7_plus/total_users)*100:.1f}%")
logger.info("")
logger.info(f"평균 활동 시간: {sum(r.get('active_seconds', 0) for r in results) / total_users / 60:.1f}분")
except KeyboardInterrupt: