From f431fdb19d77e1b7e9a1fc32909b82ce40ab5a14 Mon Sep 17 00:00:00 2001 From: Gnill82 Date: Sat, 30 Aug 2025 16:22:21 +0900 Subject: [PATCH] Enhance logging and metrics for new user analysis - Updated logging setup to include a console formatter for concise output. - Added detailed metrics for various dungeon modes (COOP, Solo, Survival, etc.) including entry counts, first results, escape counts, average survival times, and more. - Improved retention analysis with additional checks for D2 to D6 retention. - Enhanced progress logging during data collection and batch processing for better visibility. - Updated result writing to include new metrics and adjusted headers accordingly. - Refined retention funnel analysis to provide detailed insights into user retention rates across different days. --- ds_new_user_analy.py | 641 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 517 insertions(+), 124 deletions(-) diff --git a/ds_new_user_analy.py b/ds_new_user_analy.py index 91746b0..46f56b7 100644 --- a/ds_new_user_analy.py +++ b/ds_new_user_analy.py @@ -82,7 +82,7 @@ logger = None # ============================================================================== def setup_logging(log_file_path: str) -> logging.Logger: - """한국 시간 기준 로깅 설정""" + """한국 시간 기준 로깅 설정 (콘솔 간결 모드)""" class KSTFormatter(logging.Formatter): def formatTime(self, record, datefmt=None): @@ -90,6 +90,40 @@ def setup_logging(log_file_path: str) -> logging.Logger: ct = datetime.fromtimestamp(record.created, kst) return ct.strftime('%Y-%m-%dT%H:%M:%S+09:00') + class ConsoleFormatter(logging.Formatter): + """Console-optimized formatter - 간결한 콘솔 출력""" + def format(self, record): + # 콘솔에는 시간 없이 중요 메시지만 표시 + if record.levelno >= logging.WARNING: # WARNING 이상은 상세 표시 + return f"[{record.levelname}] {record.getMessage()}" + elif record.name == 'progress': # 진행률 추적용 특별 로거 + return record.getMessage() # 시간 없이 출력 + else: + # 일반 INFO 로그는 상당히 간결하게 + msg = record.getMessage() + # 주요 정보만 콘솔에 표시 + if any(keyword in msg for keyword in [ + "던전 스토커즈 신규 유저 분석", + "분석 기간", + "세션 지표 포함", + "총 신규 유저", + "분석 완료", + "분석 요약", + "리텐션 퍼널", + "D0→D1", "D1→D2", "D2→D3", "D3→D4", "D4→D5", "D5→D6", "D6→D7", + "총 소요 시간", + "Retained_d", + "평균 활동 시간", + "="*20 # 구분선 + ]): + return msg + else: + return None + + def formatMessage(self, record): + formatted = self.format(record) + return formatted if formatted is not None else "" + logger = logging.getLogger('NewUserAnalyzer') logger.setLevel(logging.INFO) @@ -97,16 +131,18 @@ def setup_logging(log_file_path: str) -> logging.Logger: for handler in logger.handlers[:]: logger.removeHandler(handler) - # 파일 핸들러 + # 파일 핸들러 (상세 로그) file_handler = logging.FileHandler(log_file_path, encoding='utf-8') file_formatter = KSTFormatter('[%(levelname)s] %(asctime)s - %(message)s') file_handler.setFormatter(file_formatter) logger.addHandler(file_handler) - # 콘솔 핸들러 + # 콘솔 핸들러 (간결 로그) console_handler = logging.StreamHandler() - console_formatter = KSTFormatter('[%(levelname)s] %(asctime)s - %(message)s') + console_formatter = ConsoleFormatter() console_handler.setFormatter(console_formatter) + # 콘솔용 필터 추가 + console_handler.addFilter(lambda record: console_formatter.format(record) is not None) logger.addHandler(console_handler) # 외부 라이브러리 로그 억제 @@ -196,11 +232,37 @@ def get_fixed_metrics_config() -> Dict[str, Dict]: "agg_type": "max_timestamp" }, - # ==================== 3.3 던전 플레이 성과 ==================== - "dungeon_entry_count": { + # ==================== 3.3 던전 플레이 성과 (모드별) ==================== + # 각 모드별 entry count + "COOP_entry_count": { "index": "ds-logs-live-survival_sta", "time_range": "d0", - "agg_type": "count" + "agg_type": "count", + "filters": [{"term": {"body.dungeon_mode.keyword": "COOP"}}] + }, + "Solo_entry_count": { + "index": "ds-logs-live-survival_sta", + "time_range": "d0", + "agg_type": "count", + "filters": [{"term": {"body.dungeon_mode.keyword": "Solo"}}] + }, + "Survival_entry_count": { + "index": "ds-logs-live-survival_sta", + "time_range": "d0", + "agg_type": "count", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival"}}] + }, + "Survival_BOT_entry_count": { + "index": "ds-logs-live-survival_sta", + "time_range": "d0", + "agg_type": "count", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival_BOT"}}] + }, + "Survival_Unprotected_entry_count": { + "index": "ds-logs-live-survival_sta", + "time_range": "d0", + "agg_type": "count", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival_Unprotected"}}] }, "dungeon_first_mode": { "index": "ds-logs-live-survival_sta", @@ -220,67 +282,292 @@ def get_fixed_metrics_config() -> Dict[str, Dict]: "agg_type": "first_value", "field": "body.result" }, - "dungeon_escape_count": { + # 각 모드별 첫 플레이 결과 + "COOP_first_result": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "first_value", + "field": "body.result", + "filters": [{"term": {"body.dungeon_mode.keyword": "COOP"}}] + }, + "Solo_first_result": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "first_value", + "field": "body.result", + "filters": [{"term": {"body.dungeon_mode.keyword": "Solo"}}] + }, + "Survival_first_result": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "first_value", + "field": "body.result", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival"}}] + }, + "Survival_BOT_first_result": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "first_value", + "field": "body.result", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival_BOT"}}] + }, + "Survival_Unprotected_first_result": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "first_value", + "field": "body.result", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival_Unprotected"}}] + }, + # 각 모드별 escape count + "COOP_escape_count": { "index": "ds-logs-live-survival_end", "time_range": "d0", "agg_type": "count", - "filters": [{"term": {"body.result": 1}}] + "filters": [{"term": {"body.result": 1}}, {"term": {"body.dungeon_mode.keyword": "COOP"}}] }, - "avg_survival_time": { + "Solo_escape_count": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "count", + "filters": [{"term": {"body.result": 1}}, {"term": {"body.dungeon_mode.keyword": "Solo"}}] + }, + "Survival_escape_count": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "count", + "filters": [{"term": {"body.result": 1}}, {"term": {"body.dungeon_mode.keyword": "Survival"}}] + }, + "Survival_BOT_escape_count": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "count", + "filters": [{"term": {"body.result": 1}}, {"term": {"body.dungeon_mode.keyword": "Survival_BOT"}}] + }, + "Survival_Unprotected_escape_count": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "count", + "filters": [{"term": {"body.result": 1}}, {"term": {"body.dungeon_mode.keyword": "Survival_Unprotected"}}] + }, + # 각 모드별 avg survival time + "COOP_avg_survival_time": { "index": "ds-logs-live-survival_end", "time_range": "d0", "agg_type": "avg", - "field": "body.play_stats.playtime" + "field": "body.play_stats.playtime", + "filters": [{"term": {"body.dungeon_mode.keyword": "COOP"}}] }, - "max_survival_time": { + "Solo_avg_survival_time": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "avg", + "field": "body.play_stats.playtime", + "filters": [{"term": {"body.dungeon_mode.keyword": "Solo"}}] + }, + "Survival_avg_survival_time": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "avg", + "field": "body.play_stats.playtime", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival"}}] + }, + "Survival_BOT_avg_survival_time": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "avg", + "field": "body.play_stats.playtime", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival_BOT"}}] + }, + "Survival_Unprotected_avg_survival_time": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "avg", + "field": "body.play_stats.playtime", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival_Unprotected"}}] + }, + # 각 모드별 max survival time + "COOP_max_survival_time": { "index": "ds-logs-live-survival_end", "time_range": "d0", "agg_type": "max", - "field": "body.play_stats.playtime" + "field": "body.play_stats.playtime", + "filters": [{"term": {"body.dungeon_mode.keyword": "COOP"}}] }, - "total_armor_break": { + "Solo_max_survival_time": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "max", + "field": "body.play_stats.playtime", + "filters": [{"term": {"body.dungeon_mode.keyword": "Solo"}}] + }, + "Survival_max_survival_time": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "max", + "field": "body.play_stats.playtime", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival"}}] + }, + "Survival_BOT_max_survival_time": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "max", + "field": "body.play_stats.playtime", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival_BOT"}}] + }, + "Survival_Unprotected_max_survival_time": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "max", + "field": "body.play_stats.playtime", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival_Unprotected"}}] + }, + # 각 모드별 raid play count + "COOP_raid_play_count": { "index": "ds-logs-live-survival_end", "time_range": "d0", "agg_type": "sum", - "field": "body.play_stats.armor_break_cnt" + "field": "body.play_stats.raid_play", + "filters": [{"term": {"body.dungeon_mode.keyword": "COOP"}}] }, - "raid_play_count": { + "Solo_raid_play_count": { "index": "ds-logs-live-survival_end", "time_range": "d0", "agg_type": "sum", - "field": "body.play_stats.raid_play" + "field": "body.play_stats.raid_play", + "filters": [{"term": {"body.dungeon_mode.keyword": "Solo"}}] }, - "escape_count": { - "index": "ds-logs-live-dead", + "Survival_raid_play_count": { + "index": "ds-logs-live-survival_end", "time_range": "d0", - "agg_type": "count", - "filters": [{"term": {"body.inter_type": 0}}] + "agg_type": "sum", + "field": "body.play_stats.raid_play", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival"}}] + }, + "Survival_BOT_raid_play_count": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "sum", + "field": "body.play_stats.raid_play", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival_BOT"}}] + }, + "Survival_Unprotected_raid_play_count": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "sum", + "field": "body.play_stats.raid_play", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival_Unprotected"}}] }, - # ==================== 3.4 전투 성과 ==================== - "monster_kill_count": { - "index": "ds-logs-live-monster_kill", + # ==================== 3.4 전투 성과 (모드별) ==================== + # 각 모드별 monster kill count + "COOP_monster_kill_count": { + "index": "ds-logs-live-survival_end", "time_range": "d0", - "agg_type": "count" + "agg_type": "sum", + "field": "body.play_stats.monster_kill_cnt", + "filters": [{"term": {"body.dungeon_mode.keyword": "COOP"}}] }, - "player_kill_count": { - "index": "ds-logs-live-player_kill", + "Solo_monster_kill_count": { + "index": "ds-logs-live-survival_end", "time_range": "d0", - "agg_type": "count", - "target_field": "uid.keyword" + "agg_type": "sum", + "field": "body.play_stats.monster_kill_cnt", + "filters": [{"term": {"body.dungeon_mode.keyword": "Solo"}}] }, - "player_killed_count": { + "Survival_monster_kill_count": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "sum", + "field": "body.play_stats.monster_kill_cnt", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival"}}] + }, + "Survival_BOT_monster_kill_count": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "sum", + "field": "body.play_stats.monster_kill_cnt", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival_BOT"}}] + }, + "Survival_Unprotected_monster_kill_count": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "sum", + "field": "body.play_stats.monster_kill_cnt", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival_Unprotected"}}] + }, + # 각 모드별 player kill count + "COOP_player_kill_count": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "sum", + "field": "body.play_stats.player_kill_cnt", + "filters": [{"term": {"body.dungeon_mode.keyword": "COOP"}}] + }, + "Solo_player_kill_count": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "sum", + "field": "body.play_stats.player_kill_cnt", + "filters": [{"term": {"body.dungeon_mode.keyword": "Solo"}}] + }, + "Survival_player_kill_count": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "sum", + "field": "body.play_stats.player_kill_cnt", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival"}}] + }, + "Survival_BOT_player_kill_count": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "sum", + "field": "body.play_stats.player_kill_cnt", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival_BOT"}}] + }, + "Survival_Unprotected_player_kill_count": { + "index": "ds-logs-live-survival_end", + "time_range": "d0", + "agg_type": "sum", + "field": "body.play_stats.player_kill_cnt", + "filters": [{"term": {"body.dungeon_mode.keyword": "Survival_Unprotected"}}] + }, + # ==================== 3.4.2 사망 원인 분석 ==================== + "death_PK": { "index": "ds-logs-live-player_kill", "time_range": "d0", "agg_type": "count", "target_field": "body.target_uid.keyword", "use_body_target": True }, - "death_count": { + "death_GiveUp": { "index": "ds-logs-live-dead", "time_range": "d0", "agg_type": "count", - "filters": [{"bool": {"must_not": {"term": {"body.inter_type": 0}}}}] + "filters": [{"term": {"body.inter_type": 0}}] + }, + "death_Mob": { + "index": "ds-logs-live-dead", + "time_range": "d0", + "agg_type": "count", + "filters": [{"term": {"body.inter_type": 1}}] + }, + "death_Trap": { + "index": "ds-logs-live-dead", + "time_range": "d0", + "agg_type": "count", + "filters": [{"term": {"body.inter_type": 10}}] + }, + "death_Red": { + "index": "ds-logs-live-dead", + "time_range": "d0", + "agg_type": "count", + "filters": [{"term": {"body.inter_type": 11}}] + }, + "death_Others": { + "index": "ds-logs-live-dead", + "time_range": "d0", + "agg_type": "count", + "filters": [{"bool": {"must_not": [{"terms": {"body.inter_type": [0, 1, 10, 11]}}]}}] }, # ==================== 3.5 진행도 및 성장 ==================== @@ -318,18 +605,8 @@ def get_fixed_metrics_config() -> Dict[str, Dict]: "agg_type": "max", "field": "body.guide_step" }, - "skill_points_earned": { - "index": "ds-logs-live-skill_point_get", - "time_range": "d0", - "agg_type": "count" - }, # ==================== 3.6 아이템 및 경제 ==================== - "items_obtained_count": { - "index": "ds-logs-live-item_get", - "time_range": "d0", - "agg_type": "count" - }, "highest_item_grade": { "index": "ds-logs-live-item_get", "time_range": "d0", @@ -516,9 +793,29 @@ def get_fixed_metrics_config() -> Dict[str, Dict]: "time_range": "d1", "agg_type": "exists" }, - "retention_check_d2_6": { + "retention_check_d2": { "index": "ds-logs-live-heartbeat", - "time_range": "d2_6", + "time_range": "d2", + "agg_type": "exists" + }, + "retention_check_d3": { + "index": "ds-logs-live-heartbeat", + "time_range": "d3", + "agg_type": "exists" + }, + "retention_check_d4": { + "index": "ds-logs-live-heartbeat", + "time_range": "d4", + "agg_type": "exists" + }, + "retention_check_d5": { + "index": "ds-logs-live-heartbeat", + "time_range": "d5", + "agg_type": "exists" + }, + "retention_check_d6": { + "index": "ds-logs-live-heartbeat", + "time_range": "d6", "agg_type": "exists" }, "retention_check_d7_plus": { @@ -582,6 +879,8 @@ def get_new_user_cohort_optimized( # 신규 생성된 유저 수집 new_user_map = {} # uid -> {'create_time': ...} page_count = 0 + total_pages_estimate = 0 + progress_logger = logging.getLogger('progress') while True: page_count += 1 @@ -590,6 +889,12 @@ def get_new_user_cohort_optimized( query["aggs"]["new_users"]["composite"]["after"] = after_key try: + # 진행률 표시 (제자리 갱신) + if page_count == 1: + progress_logger.info("\r코호트 선정: 페이지 1 처리 중...") + else: + progress_logger.info(f"\r코호트 선정: 페이지 {page_count} 처리 중... (수집: {len(new_user_map)}명)") + logger.info(f"create_uid 페이지 {page_count} 처리 중...") response = exponential_backoff_retry( client.search, @@ -614,6 +919,8 @@ def get_new_user_cohort_optimized( } logger.info(f"create_uid 페이지 {page_count}: {len(buckets)}개 처리됨") + # 콘솔에는 간결하게 + progress_logger.info(f"\r코호트 선정: 페이지 {page_count} 완료 (수집: {len(new_user_map)}명)") after_key = response["aggregations"]["new_users"].get("after_key") if not after_key: @@ -623,6 +930,8 @@ def get_new_user_cohort_optimized( logger.error(f"create_uid 처리 중 오류 (페이지 {page_count}): {e}") break + # 최종 진행률 표시 + progress_logger.info(f"\r코호트 선정 완료: 총 {len(new_user_map)}명 확인 \n") logger.info(f"총 {len(new_user_map)}명의 신규 유저 확인됨") # Step 2: 모든 create_uid 유저를 cohort에 추가 @@ -648,6 +957,7 @@ def get_new_user_cohort_optimized( } total_users += 1 + progress_logger.info(f"\r코호트 초기화 완료: {total_users}명 ") logger.info(f"cohort에 {total_users}명의 신규 유저 추가 완료") # Step 3: login_comp 인덱스에서 추가 정보 수집 (auth.id 1순위) @@ -728,6 +1038,7 @@ def get_new_user_cohort_optimized( except Exception as e: logger.error(f"login_comp 정보 수집 중 오류: {e}") + progress_logger.info(f"\rlogin_comp 정보 수집: {len(login_comp_collected)}/{total_users}명 완료 ") logger.info(f"login_comp에서 {len(login_comp_collected)}명의 정보 수집 완료") # Step 4: log_return_to_lobby 인덱스에서 차선 정보 수집 (auth.id 2순위) @@ -806,6 +1117,7 @@ def get_new_user_cohort_optimized( except Exception as e: logger.error(f"log_return_to_lobby 정보 수집 중 오류: {e}") + progress_logger.info(f"\rlobby 추가 정보 수집: {len(lobby_collected)}명 완료 ") logger.info(f"log_return_to_lobby에서 {len(lobby_collected)}명의 차선 정보 수집 완료") # 최종 통계 @@ -840,7 +1152,15 @@ def build_fixed_msearch_queries( d1_start = d0_end d1_end = (create_time_dt + timedelta(hours=48)).strftime('%Y-%m-%dT%H:%M:%SZ') d2_start = d1_end - d6_end = (create_time_dt + timedelta(hours=168)).strftime('%Y-%m-%dT%H:%M:%SZ') # 168시간 = 7일 + d2_end = (create_time_dt + timedelta(hours=72)).strftime('%Y-%m-%dT%H:%M:%SZ') + d3_start = d2_end + d3_end = (create_time_dt + timedelta(hours=96)).strftime('%Y-%m-%dT%H:%M:%SZ') + d4_start = d3_end + d4_end = (create_time_dt + timedelta(hours=120)).strftime('%Y-%m-%dT%H:%M:%SZ') + d5_start = d4_end + d5_end = (create_time_dt + timedelta(hours=144)).strftime('%Y-%m-%dT%H:%M:%SZ') + d6_start = d5_end + d6_end = (create_time_dt + timedelta(hours=168)).strftime('%Y-%m-%dT%H:%M:%SZ') d7_plus_start = d6_end for metric_name, config in metrics_config.items(): @@ -849,8 +1169,16 @@ def build_fixed_msearch_queries( time_filter = {"range": {"@timestamp": {"gte": d0_start, "lt": d0_end}}} elif config["time_range"] == "d1": time_filter = {"range": {"@timestamp": {"gte": d1_start, "lt": d1_end}}} - elif config["time_range"] == "d2_6": - time_filter = {"range": {"@timestamp": {"gte": d2_start, "lt": d6_end}}} + elif config["time_range"] == "d2": + time_filter = {"range": {"@timestamp": {"gte": d2_start, "lt": d2_end}}} + elif config["time_range"] == "d3": + time_filter = {"range": {"@timestamp": {"gte": d3_start, "lt": d3_end}}} + elif config["time_range"] == "d4": + time_filter = {"range": {"@timestamp": {"gte": d4_start, "lt": d4_end}}} + elif config["time_range"] == "d5": + time_filter = {"range": {"@timestamp": {"gte": d5_start, "lt": d5_end}}} + elif config["time_range"] == "d6": + time_filter = {"range": {"@timestamp": {"gte": d6_start, "lt": d6_end}}} elif config["time_range"] == "d7_plus": time_filter = {"range": {"@timestamp": {"gte": d7_plus_start}}} else: # 기본값 @@ -1066,7 +1394,8 @@ def process_fixed_batch( ) -> List[Dict]: """수정된 배치 처리 함수""" - logger.info(f"배치 처리 시작: {len(batch_uids)}명") + # 진행률 추적용 로거 + progress_logger = logging.getLogger('progress') try: # 1. 세션 지표 계산 (--full 옵션일 때만) @@ -1086,6 +1415,8 @@ def process_fixed_batch( msearch_queries = build_fixed_msearch_queries(batch_uids, cohort, metrics_config) body_ndjson = "\n".join(msearch_queries) + "\n" + # 콘솔에는 간단히, 파일에는 상세히 + progress_logger.info(f"\r배치 처리 중: {len(batch_uids)}명, {len(msearch_queries)//2}개 쿼리") logger.info(f"msearch 실행: {len(msearch_queries)//2}개 쿼리") msearch_responses = exponential_backoff_retry( @@ -1111,7 +1442,6 @@ def process_fixed_batch( 'nickname': user_data['nickname'], 'create_time': user_data.get('create_time_kst', 'N/A'), 'retention_status': 'Retained_d0', # 기본값, 나중에 업데이트 - 'last_active_day': 0, # 마지막 접속일 추가 'language': user_data['language'], 'country': user_data.get('country', 'N/A'), 'device': user_data['device'], @@ -1234,77 +1564,55 @@ def process_fixed_batch( result[metric_name] = 0 # 모든 메트릭 처리 후 리텐션 상태 판정 - # D7+ > D2~6 > D1 > D0 순서로 판정 (마지막 접속일 기준) + # D7+ > D6 > D5 > D4 > D3 > D2 > D1 > D0 순서로 판정 (마지막 접속일 기준) if result.get('retention_check_d7_plus', 0) > 0: result['retention_status'] = 'Retained_d7+' - result['last_active_day'] = 7 # 7+ 표시 - elif result.get('retention_check_d2_6', 0) > 0: - result['retention_status'] = 'Retained_d2~6' - result['last_active_day'] = 2 # 정확한 날짜는 알 수 없으므로 2로 표시 + elif result.get('retention_check_d6', 0) > 0: + result['retention_status'] = 'Retained_d6' + elif result.get('retention_check_d5', 0) > 0: + result['retention_status'] = 'Retained_d5' + elif result.get('retention_check_d4', 0) > 0: + result['retention_status'] = 'Retained_d4' + elif result.get('retention_check_d3', 0) > 0: + result['retention_status'] = 'Retained_d3' + elif result.get('retention_check_d2', 0) > 0: + result['retention_status'] = 'Retained_d2' elif result.get('retention_check_d1', 0) > 0: result['retention_status'] = 'Retained_d1' - result['last_active_day'] = 1 else: # D0에만 접속 (기본값 유지) result['retention_status'] = 'Retained_d0' - result['last_active_day'] = 0 - # 계산된 지표 - if result.get('dungeon_entry_count', 0) > 0: - escape_count = result.get('dungeon_escape_count', 0) - result['dungeon_escape_rate'] = round((escape_count / result['dungeon_entry_count']) * 100, 2) + # 모드별 계산된 지표 + modes = ['COOP', 'Solo', 'Survival', 'Survival_BOT', 'Survival_Unprotected'] + + for mode in modes: + # 각 모드별 escape rate 계산 + entry_key = f'{mode}_entry_count' + escape_key = f'{mode}_escape_count' - # 게임당 평균 데미지 계산을 위해 직접 쿼리 (제거된 필드들 대신) - dungeon_count = result['dungeon_entry_count'] - try: - # 시간 범위 가져오기 (create_time 기준) - create_time_dt = user_data['create_time_dt'] - d0_start = user_data['create_time_utc'] - d0_end = (create_time_dt + timedelta(hours=24)).strftime('%Y-%m-%dT%H:%M:%SZ') + if result.get(entry_key, 0) > 0: + escape_count = result.get(escape_key, 0) + result[f'{mode}_escape_rate'] = round((escape_count / result[entry_key]), 4) # 소수점 4자리 - # survival_end 인덱스에서 직접 데미지 합계 조회 - damage_query = { - "size": 0, - "query": { - "bool": { - "filter": [ - {"term": {"uid.keyword": uid}}, - {"range": {"@timestamp": {"gte": d0_start, "lt": d0_end}}} - ] - } - }, - "aggs": { - "total_monster_damage": {"sum": {"field": "body.play_stats.damage_dealt_monster"}}, - "total_player_damage": {"sum": {"field": "body.play_stats.damage_dealt_player"}} - } - } + # 각 모드별 평균 킬 수 계산 + monster_kills = result.get(f'{mode}_monster_kill_count', 0) + player_kills = result.get(f'{mode}_player_kill_count', 0) + entry_count = result[entry_key] - damage_response = exponential_backoff_retry( - client.search, - index="ds-logs-live-survival_end", - body=damage_query, - request_timeout=DEFAULT_TIMEOUT - ) - - monster_damage = damage_response.get('aggregations', {}).get('total_monster_damage', {}).get('value', 0) or 0 - player_damage = damage_response.get('aggregations', {}).get('total_player_damage', {}).get('value', 0) or 0 - - result['avg_damage_per_game_monster'] = round(monster_damage / dungeon_count, 2) - result['avg_damage_per_game_player'] = round(player_damage / dungeon_count, 2) - except Exception as e: - logger.warning(f"평균 데미지 계산 실패 (UID: {uid}): {e}") - result['avg_damage_per_game_monster'] = 0 - result['avg_damage_per_game_player'] = 0 - else: - result['dungeon_escape_rate'] = 0 - result['avg_damage_per_game_monster'] = 0 - result['avg_damage_per_game_player'] = 0 + result[f'{mode}_avg_monster_kills'] = round(monster_kills / entry_count, 2) + result[f'{mode}_avg_player_kills'] = round(player_kills / entry_count, 2) + else: + result[f'{mode}_escape_rate'] = 0.0 + result[f'{mode}_avg_monster_kills'] = 0.0 + result[f'{mode}_avg_player_kills'] = 0.0 batch_results.append(result) except Exception as e: logger.warning(f"UID '{uid}' 처리 중 오류: {e}") + progress_logger.info(f"\r배치 완료: {len(batch_results)}명 ") logger.info(f"배치 처리 완료: {len(batch_results)}명 성공") return batch_results @@ -1334,6 +1642,9 @@ def process_cohort_fixed_parallel( logger.info(f"분석 지표: {len(metrics_config)}개") logger.info(f"세션 지표 포함: {'예' if include_session_metrics else '아니오'}") + # 진행률 추적용 로거 + progress_logger = logging.getLogger('progress') + uid_list = list(cohort.keys()) chunks = [uid_list[i:i + batch_size] for i in range(0, len(uid_list), batch_size)] @@ -1346,6 +1657,7 @@ def process_cohort_fixed_parallel( for chunk in chunks } + completed_chunks = 0 with tqdm(total=len(chunks), desc="배치 처리 진행률") as pbar: for future in as_completed(future_to_chunk): chunk = future_to_chunk[future] @@ -1355,28 +1667,39 @@ def process_cohort_fixed_parallel( all_results.extend(batch_results) else: failed_chunks.append(chunk) + + completed_chunks += 1 + # 콘솔 진행률 업데이트 + progress_logger.info(f"\r병렬 처리 중: {completed_chunks}/{len(chunks)} 배치 완료, {len(all_results)}명 처리됨") + except Exception as e: logger.warning(f"배치 처리 실패: {e}") failed_chunks.append(chunk) + completed_chunks += 1 + progress_logger.info(f"\r병렬 처리 중: {completed_chunks}/{len(chunks)} 배치 완료 (오류 1개), {len(all_results)}명 처리됨") finally: pbar.update(1) # 실패한 청크 재처리 if failed_chunks: logger.info(f"실패한 {len(failed_chunks)}개 배치 재처리 중...") - for chunk in failed_chunks: + progress_logger.info(f"\r재처리 중: {len(failed_chunks)}개 배치...") + for i, chunk in enumerate(failed_chunks): try: batch_results = process_fixed_batch(client, chunk, cohort, metrics_config, include_session_metrics) all_results.extend(batch_results) + progress_logger.info(f"\r재처리 중: {i+1}/{len(failed_chunks)} 완료") except Exception as e: logger.error(f"재처리 실패: {e}") + # 최종 결과 + progress_logger.info(f"\r병렬 처리 완료: 총 {len(all_results)}명 성공 \\n") logger.info(f"2단계 완료: {len(all_results)}명 처리 성공") logger.info("=" * 80) return all_results -def write_fixed_results(results: List[Dict], output_path: str) -> None: +def write_fixed_results(results: List[Dict], output_path: str, include_session_metrics: bool = False) -> None: """수정된 결과 저장 (retention_d1 제거)""" logger.info("=" * 80) @@ -1387,17 +1710,38 @@ def write_fixed_results(results: List[Dict], output_path: str) -> None: return # 수정된 헤더 (first_login_time 제거, create_time만 사용, country 및 last_active_day 추가) + # 기본 헤더 headers = [ - 'uid', 'auth_id', 'nickname', 'create_time', 'retention_status', 'last_active_day', 'language', 'country', 'device', - 'active_seconds', 'total_playtime_minutes', 'session_count', 'avg_session_length', 'logout_abnormal', - 'dungeon_entry_count', 'dungeon_first_mode', 'dungeon_first_stalker', 'dungeon_first_result', - 'dungeon_escape_count', 'dungeon_escape_rate', 'avg_survival_time', 'max_survival_time', - 'total_armor_break', 'raid_play_count', 'escape_count', - 'monster_kill_count', 'player_kill_count', 'player_killed_count', 'death_count', - 'avg_damage_per_game_monster', 'avg_damage_per_game_player', + 'uid', 'auth_id', 'nickname', 'create_time', 'retention_status', 'language', 'country', 'device', + 'dungeon_first_mode', 'dungeon_first_stalker', 'dungeon_first_result', + 'death_PK', 'death_GiveUp', 'death_Mob', 'death_Trap', 'death_Red', 'death_Others' + ] + + # 세션 지표 (옵션에 따른 조건부 포함) + if include_session_metrics: + session_headers = ['active_seconds', 'total_playtime_minutes', 'session_count', 'avg_session_length', 'logout_abnormal'] + headers = headers[:4] + session_headers + headers[4:] # retention_status 뒤에 삽입 + + # 각 모드별 헤더 추가 (삭제된 지표 제외) + modes = ['COOP', 'Solo', 'Survival', 'Survival_BOT', 'Survival_Unprotected'] + for mode in modes: + headers.extend([ + f'{mode}_entry_count', + f'{mode}_escape_count', + f'{mode}_escape_rate', + f'{mode}_avg_survival_time', + f'{mode}_max_survival_time', + f'{mode}_raid_play_count', + f'{mode}_first_result', + f'{mode}_avg_monster_kills', + f'{mode}_avg_player_kills' + ]) + + # 나머지 헤더 (삭제된 지표 제외) + headers.extend([ 'level_max', 'level_max_stalker', 'tutorial_entry', 'tutorial_completed', - 'guide_quest_stage', 'skill_points_earned', - 'items_obtained_count', 'highest_item_grade', 'blueprint_use_count', 'shop_buy_count', + 'guide_quest_stage', + 'highest_item_grade', 'blueprint_use_count', 'shop_buy_count', 'shop_sell_count', 'gold_spent', 'gold_earned', 'storage_in_count', 'storage_out_count', 'enchant_count', 'enchant_gold_spent', 'ingame_equip_count', 'object_interaction_count', @@ -1406,7 +1750,7 @@ def write_fixed_results(results: List[Dict], output_path: str) -> None: 'exchange_register_count', 'exchange_use_count', 'coupon_used', 'button_click_count', 'hideout_upgrade_count', 'hideout_max_level', 'season_pass_buy', 'season_pass_max_step', 'last_logout_time' - ] + ]) try: with open(output_path, 'w', newline='', encoding='utf-8-sig') as csvfile: @@ -1499,24 +1843,73 @@ def main(): client, cohort, args.batch_size, args.max_workers, metrics_config, args.full ) - write_fixed_results(results, str(csv_file_path)) + write_fixed_results(results, str(csv_file_path), args.full) # 요약 if results: total_users = len(results) - retained_d0 = sum(1 for r in results if r.get('retention_status') == 'Retained_d0') - retained_d1 = sum(1 for r in results if r.get('retention_status') == 'Retained_d1') - retained_d2_6 = sum(1 for r in results if r.get('retention_status') == 'Retained_d2~6') - retained_d7_plus = sum(1 for r in results if r.get('retention_status') == 'Retained_d7+') + retention_counts = {} + retention_groups = ['Retained_d0', 'Retained_d1', 'Retained_d2', 'Retained_d3', 'Retained_d4', 'Retained_d5', 'Retained_d6', 'Retained_d7+'] + + # 각 그룹별 카운트 + for group in retention_groups: + retention_counts[group] = sum(1 for r in results if r.get('retention_status') == group) logger.info("=" * 80) logger.info("분석 요약") logger.info("=" * 80) logger.info(f"총 신규 유저: {total_users:,}명") - logger.info(f"D+0 리텐션: {retained_d0:,}명 ({(retained_d0/total_users)*100:.1f}%)") - logger.info(f"D+1 리텐션: {retained_d1:,}명 ({(retained_d1/total_users)*100:.1f}%)") - logger.info(f"D+2~6 리텐션: {retained_d2_6:,}명 ({(retained_d2_6/total_users)*100:.1f}%)") - logger.info(f"D+7+ 리텐션: {retained_d7_plus:,}명 ({(retained_d7_plus/total_users)*100:.1f}%)") + + # 기본 리텐션 분포 + for group in retention_groups: + count = retention_counts[group] + percentage = (count / total_users) * 100 + logger.info(f"{group}: {count:,}명 ({percentage:.1f}%)") + + # 리텐션 퍼널 요약 (단계별 + 누적) + logger.info("") + logger.info("=" * 50) + logger.info("리텐션 퍼널 분석") + logger.info("=" * 50) + + # 누적 리텐션 계산 (D1 이상 접속한 유저들) + cumulative_d1_plus = sum(retention_counts[group] for group in retention_groups[1:]) # D1부터 + cumulative_d2_plus = sum(retention_counts[group] for group in retention_groups[2:]) # D2부터 + cumulative_d3_plus = sum(retention_counts[group] for group in retention_groups[3:]) # D3부터 + cumulative_d4_plus = sum(retention_counts[group] for group in retention_groups[4:]) # D4부터 + cumulative_d5_plus = sum(retention_counts[group] for group in retention_groups[5:]) # D5부터 + cumulative_d6_plus = sum(retention_counts[group] for group in retention_groups[6:]) # D6부터 + cumulative_d7_plus = retention_counts['Retained_d7+'] + + # 퍼널 분석 (단계별 잔존율 + 누적 리텐션율) + if cumulative_d1_plus > 0: + logger.info(f"D0→D1: {(cumulative_d1_plus/total_users)*100:.1f}% ({cumulative_d1_plus:,}/{total_users:,}명) | 누적: {(cumulative_d1_plus/total_users)*100:.1f}%") + + if cumulative_d2_plus > 0: + d1_to_d2_rate = (cumulative_d2_plus/cumulative_d1_plus)*100 + logger.info(f"D1→D2: {d1_to_d2_rate:.1f}% ({cumulative_d2_plus:,}/{cumulative_d1_plus:,}명) | 누적: {(cumulative_d2_plus/total_users)*100:.1f}%") + + if cumulative_d3_plus > 0: + d2_to_d3_rate = (cumulative_d3_plus/cumulative_d2_plus)*100 + logger.info(f"D2→D3: {d2_to_d3_rate:.1f}% ({cumulative_d3_plus:,}/{cumulative_d2_plus:,}명) | 누적: {(cumulative_d3_plus/total_users)*100:.1f}%") + + if cumulative_d4_plus > 0: + d3_to_d4_rate = (cumulative_d4_plus/cumulative_d3_plus)*100 + logger.info(f"D3→D4: {d3_to_d4_rate:.1f}% ({cumulative_d4_plus:,}/{cumulative_d3_plus:,}명) | 누적: {(cumulative_d4_plus/total_users)*100:.1f}%") + + if cumulative_d5_plus > 0: + d4_to_d5_rate = (cumulative_d5_plus/cumulative_d4_plus)*100 + logger.info(f"D4→D5: {d4_to_d5_rate:.1f}% ({cumulative_d5_plus:,}/{cumulative_d4_plus:,}명) | 누적: {(cumulative_d5_plus/total_users)*100:.1f}%") + + if cumulative_d6_plus > 0: + d5_to_d6_rate = (cumulative_d6_plus/cumulative_d5_plus)*100 + logger.info(f"D5→D6: {d5_to_d6_rate:.1f}% ({cumulative_d6_plus:,}/{cumulative_d5_plus:,}명) | 누적: {(cumulative_d6_plus/total_users)*100:.1f}%") + + if cumulative_d7_plus > 0: + d6_to_d7_rate = (cumulative_d7_plus/cumulative_d6_plus)*100 + logger.info(f"D6→D7+: {d6_to_d7_rate:.1f}% ({cumulative_d7_plus:,}/{cumulative_d6_plus:,}명) | 누적: {(cumulative_d7_plus/total_users)*100:.1f}%") + + logger.info("") logger.info(f"평균 활동 시간: {sum(r.get('active_seconds', 0) for r in results) / total_users / 60:.1f}분") except KeyboardInterrupt: