From 30c66b2bd41f964761fa308412e0f3a5b03f5f1d Mon Sep 17 00:00:00 2001 From: Hexeong <123macanic@naver.com> Date: Tue, 18 Aug 2026 16:11:12 +0900 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20MySQL=20=EB=B0=B1=EC=97=85=20?= =?UTF-8?q?=EC=8B=A4=ED=8C=A8=20=EC=95=8C=EB=A6=BC=20=EB=82=B4=EB=B6=80=20?= =?UTF-8?q?=EC=A0=84=EC=9A=A9=20API=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - DB EC2 의 백업 실패 이벤트를 받아 Discord 로 중계하는 내부 전용 API 를 추가한다 - 공유 토큰 헤더로 호출자를 인증하고, 토큰이 설정되지 않으면 모든 요청을 거부한다 - 같은 유형과 인스턴스의 알림을 10분간 억제하고, 전송에 실패하면 억제를 해제한다 - 담당 역할만 멘션하고 everyone, here 멘션은 차단한다 - DiscordNotifier 의 전송 로직을 DiscordWebhookSender 로 분리해 채널별로 재사용한다 Co-Authored-By: Claude Opus 5 --- .../alarm/config/DbBackupAlarmProperties.java | 11 + .../config/InternalAlarmAuthProperties.java | 10 + .../controller/DbBackupAlarmController.java | 32 +++ .../alarm/domain/DbBackupAlarmType.java | 19 ++ .../alarm/dto/DbBackupAlarmRequest.java | 25 ++ .../alarm/service/DbBackupAlarmService.java | 141 +++++++++++ .../common/discord/DiscordNotifier.java | 18 +- .../common/discord/DiscordWebhookSender.java | 59 +++++ .../common/exception/ErrorCode.java | 3 + .../config/SecurityConfiguration.java | 2 + .../alarm/dto/DbBackupAlarmRequestTest.java | 155 ++++++++++++ .../service/DbBackupAlarmServiceTest.java | 235 ++++++++++++++++++ ...kupAlarmServiceTokenNotConfiguredTest.java | 53 ++++ .../discord/DiscordWebhookSenderTest.java | 130 ++++++++++ 14 files changed, 877 insertions(+), 16 deletions(-) create mode 100644 src/main/java/com/example/solidconnection/alarm/config/DbBackupAlarmProperties.java create mode 100644 src/main/java/com/example/solidconnection/alarm/config/InternalAlarmAuthProperties.java create mode 100644 src/main/java/com/example/solidconnection/alarm/controller/DbBackupAlarmController.java create mode 100644 src/main/java/com/example/solidconnection/alarm/domain/DbBackupAlarmType.java create mode 100644 src/main/java/com/example/solidconnection/alarm/dto/DbBackupAlarmRequest.java create mode 100644 src/main/java/com/example/solidconnection/alarm/service/DbBackupAlarmService.java create mode 100644 src/main/java/com/example/solidconnection/common/discord/DiscordWebhookSender.java create mode 100644 src/test/java/com/example/solidconnection/alarm/dto/DbBackupAlarmRequestTest.java create mode 100644 src/test/java/com/example/solidconnection/alarm/service/DbBackupAlarmServiceTest.java create mode 100644 src/test/java/com/example/solidconnection/alarm/service/DbBackupAlarmServiceTokenNotConfiguredTest.java create mode 100644 src/test/java/com/example/solidconnection/common/discord/DiscordWebhookSenderTest.java diff --git a/src/main/java/com/example/solidconnection/alarm/config/DbBackupAlarmProperties.java b/src/main/java/com/example/solidconnection/alarm/config/DbBackupAlarmProperties.java new file mode 100644 index 000000000..af6f8ac85 --- /dev/null +++ b/src/main/java/com/example/solidconnection/alarm/config/DbBackupAlarmProperties.java @@ -0,0 +1,11 @@ +package com.example.solidconnection.alarm.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "discord.db-backup-fail-alarm") +public record DbBackupAlarmProperties( + String webhookUrl, + String mentionRoleId +) { + +} diff --git a/src/main/java/com/example/solidconnection/alarm/config/InternalAlarmAuthProperties.java b/src/main/java/com/example/solidconnection/alarm/config/InternalAlarmAuthProperties.java new file mode 100644 index 000000000..f8bf93677 --- /dev/null +++ b/src/main/java/com/example/solidconnection/alarm/config/InternalAlarmAuthProperties.java @@ -0,0 +1,10 @@ +package com.example.solidconnection.alarm.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "internal-alarm") +public record InternalAlarmAuthProperties( + String token +) { + +} diff --git a/src/main/java/com/example/solidconnection/alarm/controller/DbBackupAlarmController.java b/src/main/java/com/example/solidconnection/alarm/controller/DbBackupAlarmController.java new file mode 100644 index 000000000..a1bc700b8 --- /dev/null +++ b/src/main/java/com/example/solidconnection/alarm/controller/DbBackupAlarmController.java @@ -0,0 +1,32 @@ +package com.example.solidconnection.alarm.controller; + +import com.example.solidconnection.alarm.dto.DbBackupAlarmRequest; +import com.example.solidconnection.alarm.service.DbBackupAlarmService; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/internal/alarms") +@RequiredArgsConstructor +public class DbBackupAlarmController { + + private static final String INTERNAL_ALARM_TOKEN_HEADER = "X-Internal-Alarm-Token"; + + private final DbBackupAlarmService dbBackupAlarmService; + + // DB EC2 의 백업 실패 이벤트를 받아 Discord 로 알리는 내부 전용 api + @PostMapping("/db-backup") + public ResponseEntity alarmBackupFailure( + @RequestHeader(value = INTERNAL_ALARM_TOKEN_HEADER, required = false) String token, + @Valid @RequestBody DbBackupAlarmRequest dbBackupAlarmRequest + ) { + dbBackupAlarmService.alarmBackupFailure(token, dbBackupAlarmRequest); + return ResponseEntity.accepted().build(); + } +} diff --git a/src/main/java/com/example/solidconnection/alarm/domain/DbBackupAlarmType.java b/src/main/java/com/example/solidconnection/alarm/domain/DbBackupAlarmType.java new file mode 100644 index 000000000..e5ad9bf42 --- /dev/null +++ b/src/main/java/com/example/solidconnection/alarm/domain/DbBackupAlarmType.java @@ -0,0 +1,19 @@ +package com.example.solidconnection.alarm.domain; + +import lombok.Getter; + +@Getter +public enum DbBackupAlarmType { + + DUMP_FAILED("전체 덤프 실패"), + BINLOG_UPLOAD_FAILED("바이너리 로그 업로드 실패"), + BINLOG_GAP_DETECTED("바이너리 로그 누락"), + BINLOG_UPLOAD_DELAYED("바이너리 로그 업로드 지연"), + ; + + private final String displayName; + + DbBackupAlarmType(String displayName) { + this.displayName = displayName; + } +} diff --git a/src/main/java/com/example/solidconnection/alarm/dto/DbBackupAlarmRequest.java b/src/main/java/com/example/solidconnection/alarm/dto/DbBackupAlarmRequest.java new file mode 100644 index 000000000..922da3db7 --- /dev/null +++ b/src/main/java/com/example/solidconnection/alarm/dto/DbBackupAlarmRequest.java @@ -0,0 +1,25 @@ +package com.example.solidconnection.alarm.dto; + +import com.example.solidconnection.alarm.domain.DbBackupAlarmType; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Size; +import java.time.Instant; + +public record DbBackupAlarmRequest( + + @NotNull + DbBackupAlarmType type, + + @NotBlank + @Size(max = 64) + String instanceId, + + @NotNull + Instant occurredAt, + + @Size(max = 1000) + String detail +) { + +} diff --git a/src/main/java/com/example/solidconnection/alarm/service/DbBackupAlarmService.java b/src/main/java/com/example/solidconnection/alarm/service/DbBackupAlarmService.java new file mode 100644 index 000000000..f6c41e473 --- /dev/null +++ b/src/main/java/com/example/solidconnection/alarm/service/DbBackupAlarmService.java @@ -0,0 +1,141 @@ +package com.example.solidconnection.alarm.service; + +import static com.example.solidconnection.common.exception.ErrorCode.INTERNAL_ALARM_UNAUTHORIZED; + +import com.example.solidconnection.alarm.config.DbBackupAlarmProperties; +import com.example.solidconnection.alarm.config.InternalAlarmAuthProperties; +import com.example.solidconnection.alarm.dto.DbBackupAlarmRequest; +import com.example.solidconnection.common.discord.DiscordWebhookSender; +import com.example.solidconnection.common.exception.CustomException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.time.Duration; +import java.util.List; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Service; + +/* + * - DB EC2 는 private subnet 에 있어 Discord 로 직접 요청할 수 없다. + * - 따라서 백업 실패 이벤트를 전달받아 Discord 로 중계한다. + * */ +@Service +@RequiredArgsConstructor +@Slf4j +public class DbBackupAlarmService { + + private static final String SUPPRESSION_KEY_PREFIX = "db-backup-alarm:"; + private static final Duration SUPPRESSION_TTL = Duration.ofMinutes(10); + private static final String ROLE_MENTION_FORMAT = "<@&%s>"; + private static final String EMPTY_DETAIL = "-"; + + private final DiscordWebhookSender discordWebhookSender; + private final DbBackupAlarmProperties dbBackupAlarmProperties; + private final InternalAlarmAuthProperties internalAlarmAuthProperties; + private final RedisTemplate redisTemplate; + + @Value("${spring.profiles.active:}") + private String environment; + + public void alarmBackupFailure(String token, DbBackupAlarmRequest request) { + validateToken(token); + + String suppressionKey = buildSuppressionKey(request); + if (isSuppressed(suppressionKey)) { + return; + } + boolean isSent = discordWebhookSender.send( + dbBackupAlarmProperties.webhookUrl(), + buildMessage(request), + mentionableRoleIds() + ); + if (!isSent) { + releaseSuppression(suppressionKey); + } + } + + /* + * - 토큰이 설정되지 않은 환경에서는 모든 요청을 거부한다. + * - 설정 누락과 토큰 불일치를 같은 응답으로 처리해 내부 상태가 드러나지 않게 한다. + * */ + private void validateToken(String token) { + String configuredToken = internalAlarmAuthProperties.token(); + if (configuredToken == null || configuredToken.isBlank()) { + log.error("내부 알림 인증 토큰이 설정되지 않아 요청을 거부했습니다."); + throw new CustomException(INTERNAL_ALARM_UNAUTHORIZED); + } + if (token == null || !MessageDigest.isEqual( + token.getBytes(StandardCharsets.UTF_8), + configuredToken.getBytes(StandardCharsets.UTF_8))) { + throw new CustomException(INTERNAL_ALARM_UNAUTHORIZED); + } + } + + private String buildSuppressionKey(DbBackupAlarmRequest request) { + return SUPPRESSION_KEY_PREFIX + request.type().name() + ":" + request.instanceId(); + } + + /* + * - 같은 유형과 인스턴스의 알림이 반복되면 일정 시간 동안 전송하지 않는다. + * - Redis 를 사용할 수 없을 때는 알림 누락을 막기 위해 억제하지 않는다. + * */ + private boolean isSuppressed(String suppressionKey) { + try { + Boolean isFirstAlarm = redisTemplate.opsForValue().setIfAbsent(suppressionKey, "1", SUPPRESSION_TTL); + return !Boolean.TRUE.equals(isFirstAlarm); + } catch (Exception e) { + log.error("백업 알림 중복 억제 상태를 확인하지 못해 알림을 그대로 전송합니다. key={}", suppressionKey, e); + return false; + } + } + + private String buildMessage(DbBackupAlarmRequest request) { + return buildRoleMention() + "[%s] MySQL 백업 알림: %s\n인스턴스: %s\n발생 시각: %s\n상세: %s" + .formatted( + environment.toUpperCase(), + request.type().getDisplayName(), + request.instanceId(), + request.occurredAt(), + resolveDetail(request.detail()) + ); + } + + /* + * - 멘션할 역할이 설정되지 않으면 멘션 없이 알림만 보낸다. + * */ + private String buildRoleMention() { + String mentionRoleId = dbBackupAlarmProperties.mentionRoleId(); + if (mentionRoleId == null || mentionRoleId.isBlank()) { + return ""; + } + return ROLE_MENTION_FORMAT.formatted(mentionRoleId) + "\n"; + } + + private String resolveDetail(String detail) { + if (detail == null || detail.isBlank()) { + return EMPTY_DETAIL; + } + return detail; + } + + private List mentionableRoleIds() { + String mentionRoleId = dbBackupAlarmProperties.mentionRoleId(); + if (mentionRoleId == null || mentionRoleId.isBlank()) { + return List.of(); + } + return List.of(mentionRoleId); + } + + /* + * - 전송에 실패하면 억제 상태를 되돌려 다음 백업 주기의 알림이 막히지 않게 한다. + * */ + private void releaseSuppression(String suppressionKey) { + try { + redisTemplate.delete(suppressionKey); + } catch (Exception e) { + log.error("백업 알림 중복 억제 상태를 해제하지 못했습니다. key={}", suppressionKey, e); + } + } +} diff --git a/src/main/java/com/example/solidconnection/common/discord/DiscordNotifier.java b/src/main/java/com/example/solidconnection/common/discord/DiscordNotifier.java index 8f4926f1d..d40c7e138 100644 --- a/src/main/java/com/example/solidconnection/common/discord/DiscordNotifier.java +++ b/src/main/java/com/example/solidconnection/common/discord/DiscordNotifier.java @@ -1,26 +1,19 @@ package com.example.solidconnection.common.discord; -import java.util.Map; import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; import org.springframework.scheduling.annotation.Async; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.stereotype.Component; -import org.springframework.web.client.RestTemplate; @Component @RequiredArgsConstructor @EnableAsync -@Slf4j public class DiscordNotifier { private static final String ADMIN_PAGE_URL = "https://admins.solid-connection.com"; - private final RestTemplate restTemplate; + private final DiscordWebhookSender discordWebhookSender; @Value("${discord.webhook-url:}") private String webhookUrl; @@ -33,14 +26,7 @@ public void notify(DiscordNotificationType type, String applicantInfo) { if (webhookUrl.isBlank()) { return; } - try { - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); - HttpEntity> request = new HttpEntity<>(Map.of("content", buildMessage(type, applicantInfo)), headers); - restTemplate.postForEntity(webhookUrl, request, Void.class); - } catch (Exception e) { - log.error("Discord 검수 알림 전송 실패. type={}, applicantInfo={}", type, applicantInfo, e); - } + discordWebhookSender.send(webhookUrl, buildMessage(type, applicantInfo)); } private String buildMessage(DiscordNotificationType type, String applicantInfo) { diff --git a/src/main/java/com/example/solidconnection/common/discord/DiscordWebhookSender.java b/src/main/java/com/example/solidconnection/common/discord/DiscordWebhookSender.java new file mode 100644 index 000000000..0a7ca3dd5 --- /dev/null +++ b/src/main/java/com/example/solidconnection/common/discord/DiscordWebhookSender.java @@ -0,0 +1,59 @@ +package com.example.solidconnection.common.discord; + +import java.util.List; +import java.util.Map; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +/* + * - Discord Webhook 으로 메시지를 전송한다. + * - 알림 전송 실패가 호출한 기능을 실패시키지 않도록 예외를 격리하고, 전송 여부를 반환해 후속 처리를 맡긴다. + * - 채널별로 webhook url 이 다르므로 url 을 인자로 받는다. + * */ +@Component +@RequiredArgsConstructor +@Slf4j +public class DiscordWebhookSender { + + private final RestTemplate restTemplate; + + public boolean send(String webhookUrl, String content) { + return send(webhookUrl, content, List.of()); + } + + /* + * - mentionableRoleIds 에 지정한 역할만 멘션할 수 있다. + * - @everyone 과 @here 는 항상 차단되므로 content 에 섞여 들어와도 채널 전체를 호출하지 않는다. + * */ + public boolean send(String webhookUrl, String content, List mentionableRoleIds) { + if (webhookUrl == null || webhookUrl.isBlank()) { + log.error("Discord webhook url 이 설정되지 않아 알림을 전송하지 못했습니다."); + return false; + } + try { + restTemplate.postForEntity(webhookUrl, buildRequest(content, mentionableRoleIds), Void.class); + return true; + } catch (Exception e) { + log.error("Discord 알림 전송에 실패했습니다.", e); + return false; + } + } + + private HttpEntity> buildRequest(String content, List mentionableRoleIds) { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + Map body = Map.of( + "content", content, + "allowed_mentions", Map.of( + "parse", List.of(), + "roles", mentionableRoleIds + ) + ); + return new HttpEntity<>(body, headers); + } +} diff --git a/src/main/java/com/example/solidconnection/common/exception/ErrorCode.java b/src/main/java/com/example/solidconnection/common/exception/ErrorCode.java index 75180f9b0..ebf0e0102 100644 --- a/src/main/java/com/example/solidconnection/common/exception/ErrorCode.java +++ b/src/main/java/com/example/solidconnection/common/exception/ErrorCode.java @@ -191,6 +191,9 @@ public enum ErrorCode { // import INVALID_MARKDOWN_FORMAT(HttpStatus.BAD_REQUEST.value(), "올바른 마크다운 표 형식이 아닙니다."), + // internal alarm + INTERNAL_ALARM_UNAUTHORIZED(HttpStatus.UNAUTHORIZED.value(), "요청을 인증할 수 없습니다."), + // general JSON_PARSING_FAILED(HttpStatus.BAD_REQUEST.value(), "JSON 파싱을 할 수 없습니다."), JWT_EXCEPTION(HttpStatus.BAD_REQUEST.value(), "JWT 토큰을 처리할 수 없습니다."), diff --git a/src/main/java/com/example/solidconnection/security/config/SecurityConfiguration.java b/src/main/java/com/example/solidconnection/security/config/SecurityConfiguration.java index 26acbe59a..8c43468de 100644 --- a/src/main/java/com/example/solidconnection/security/config/SecurityConfiguration.java +++ b/src/main/java/com/example/solidconnection/security/config/SecurityConfiguration.java @@ -65,6 +65,8 @@ public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { .requestMatchers("/connect/**").authenticated() .requestMatchers("/admin/auth/**").permitAll() .requestMatchers("/admin/**").hasRole(ADMIN.name()) + // 내부 전용 경로는 사용자 토큰이 없는 인프라가 호출하므로 공유 토큰으로 직접 인증하고, 외부 접근은 nginx 에서 차단한다. + .requestMatchers("/internal/**").permitAll() .anyRequest().permitAll() ) .exceptionHandling(exception -> exception diff --git a/src/test/java/com/example/solidconnection/alarm/dto/DbBackupAlarmRequestTest.java b/src/test/java/com/example/solidconnection/alarm/dto/DbBackupAlarmRequestTest.java new file mode 100644 index 000000000..7f72edc90 --- /dev/null +++ b/src/test/java/com/example/solidconnection/alarm/dto/DbBackupAlarmRequestTest.java @@ -0,0 +1,155 @@ +package com.example.solidconnection.alarm.dto; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertAll; + +import com.example.solidconnection.alarm.domain.DbBackupAlarmType; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import jakarta.validation.ConstraintViolation; +import jakarta.validation.Validation; +import jakarta.validation.Validator; +import jakarta.validation.ValidatorFactory; +import java.time.Instant; +import java.util.Set; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +@DisplayName("DB 백업 알림 요청 테스트") +class DbBackupAlarmRequestTest { + + private static final String INSTANCE_ID = "i-0c5d57927ef9ca426"; + private static final Instant OCCURRED_AT = Instant.parse("2026-08-17T03:00:00Z"); + + private Validator validator; + private ObjectMapper objectMapper; + + @BeforeEach + void setUp() { + ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); + validator = factory.getValidator(); + objectMapper = JsonMapper.builder() + .addModule(new JavaTimeModule()) + .build(); + } + + @Nested + @DisplayName("백업 스크립트가 보내는 요청 역직렬화") + class 요청을_역직렬화한다 { + + @Test + void 백업_스크립트의_json_을_역직렬화한다() throws Exception { + // given + String json = """ + { + "type": "BINLOG_GAP_DETECTED", + "instanceId": "i-0c5d57927ef9ca426", + "occurredAt": "2026-08-17T03:00:00Z", + "detail": "binlog.000012 와 binlog.000014 사이 누락" + }"""; + + // when + DbBackupAlarmRequest request = objectMapper.readValue(json, DbBackupAlarmRequest.class); + + // then + assertAll( + () -> assertThat(request.type()).isEqualTo(DbBackupAlarmType.BINLOG_GAP_DETECTED), + () -> assertThat(request.instanceId()).isEqualTo(INSTANCE_ID), + () -> assertThat(request.occurredAt()).isEqualTo(OCCURRED_AT), + () -> assertThat(request.detail()).contains("binlog.000012") + ); + } + + @Test + void 상세_내용이_없는_json_도_역직렬화한다() throws Exception { + // given + String json = """ + { + "type": "DUMP_FAILED", + "instanceId": "i-0c5d57927ef9ca426", + "occurredAt": "2026-08-17T03:00:00Z" + }"""; + + // when + DbBackupAlarmRequest request = objectMapper.readValue(json, DbBackupAlarmRequest.class); + + // then + assertAll( + () -> assertThat(request.type()).isEqualTo(DbBackupAlarmType.DUMP_FAILED), + () -> assertThat(request.detail()).isNull() + ); + } + } + + @Nested + @DisplayName("요청 유효성 검증") + class 요청을_검증한다 { + + @Test + void 필수_값이_모두_있으면_검증을_통과한다() { + // given + DbBackupAlarmRequest request = new DbBackupAlarmRequest( + DbBackupAlarmType.DUMP_FAILED, INSTANCE_ID, OCCURRED_AT, null); + + // when + Set> violations = validator.validate(request); + + // then + assertThat(violations).isEmpty(); + } + + @Test + void 알림_유형이_없으면_검증에_실패한다() { + // given + DbBackupAlarmRequest request = new DbBackupAlarmRequest(null, INSTANCE_ID, OCCURRED_AT, null); + + // when + Set> violations = validator.validate(request); + + // then + assertThat(violations).hasSize(1); + } + + @Test + void 인스턴스_식별자가_비어_있으면_검증에_실패한다() { + // given + DbBackupAlarmRequest request = new DbBackupAlarmRequest( + DbBackupAlarmType.DUMP_FAILED, " ", OCCURRED_AT, null); + + // when + Set> violations = validator.validate(request); + + // then + assertThat(violations).hasSize(1); + } + + @Test + void 발생_시각이_없으면_검증에_실패한다() { + // given + DbBackupAlarmRequest request = new DbBackupAlarmRequest( + DbBackupAlarmType.DUMP_FAILED, INSTANCE_ID, null, null); + + // when + Set> violations = validator.validate(request); + + // then + assertThat(violations).hasSize(1); + } + + @Test + void 상세_내용이_1000자를_넘으면_검증에_실패한다() { + // given + DbBackupAlarmRequest request = new DbBackupAlarmRequest( + DbBackupAlarmType.DUMP_FAILED, INSTANCE_ID, OCCURRED_AT, "e".repeat(1001)); + + // when + Set> violations = validator.validate(request); + + // then + assertThat(violations).hasSize(1); + } + } +} diff --git a/src/test/java/com/example/solidconnection/alarm/service/DbBackupAlarmServiceTest.java b/src/test/java/com/example/solidconnection/alarm/service/DbBackupAlarmServiceTest.java new file mode 100644 index 000000000..0e432eb94 --- /dev/null +++ b/src/test/java/com/example/solidconnection/alarm/service/DbBackupAlarmServiceTest.java @@ -0,0 +1,235 @@ +package com.example.solidconnection.alarm.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.example.solidconnection.alarm.domain.DbBackupAlarmType; +import com.example.solidconnection.alarm.dto.DbBackupAlarmRequest; +import com.example.solidconnection.common.discord.DiscordWebhookSender; +import com.example.solidconnection.common.exception.CustomException; +import com.example.solidconnection.support.TestContainerSpringBootTest; +import java.time.Instant; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.bean.override.mockito.MockitoBean; + +@TestContainerSpringBootTest +@TestPropertySource(properties = { + "internal-alarm.token=test-internal-alarm-token", + "discord.db-backup-fail-alarm.webhook-url=https://discord.test/webhooks/db-backup", + "discord.db-backup-fail-alarm.mention-role-id=1234567890" +}) +@DisplayName("DB 백업 알림 서비스 테스트") +class DbBackupAlarmServiceTest { + + private static final String VALID_TOKEN = "test-internal-alarm-token"; + private static final String WEBHOOK_URL = "https://discord.test/webhooks/db-backup"; + private static final String MENTION_ROLE_ID = "1234567890"; + private static final String INSTANCE_ID = "i-0c5d57927ef9ca426"; + private static final Instant OCCURRED_AT = Instant.parse("2026-08-17T03:00:00Z"); + + @Autowired + private DbBackupAlarmService dbBackupAlarmService; + + @MockitoBean + private DiscordWebhookSender discordWebhookSender; + + @BeforeEach + void setUp() { + reset(discordWebhookSender); + when(discordWebhookSender.send(anyString(), anyString(), anyList())).thenReturn(true); + } + + private DbBackupAlarmRequest 백업_알림_요청(DbBackupAlarmType type, String instanceId) { + return new DbBackupAlarmRequest(type, instanceId, OCCURRED_AT, "exit code 1"); + } + + @Nested + @DisplayName("백업 실패 알림 전송") + class 백업_실패_알림을_전송한다 { + + @Test + void 유효한_토큰으로_요청하면_설정된_webhook_으로_알림을_전송한다() { + // given + DbBackupAlarmRequest request = 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, INSTANCE_ID); + + // when + dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request); + + // then + ArgumentCaptor urlCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor contentCaptor = ArgumentCaptor.forClass(String.class); + verify(discordWebhookSender).send(urlCaptor.capture(), contentCaptor.capture(), anyList()); + assertAll( + () -> assertThat(urlCaptor.getValue()).isEqualTo(WEBHOOK_URL), + () -> assertThat(contentCaptor.getValue()).contains(DbBackupAlarmType.DUMP_FAILED.getDisplayName()), + () -> assertThat(contentCaptor.getValue()).contains(INSTANCE_ID), + () -> assertThat(contentCaptor.getValue()).contains("2026-08-17T03:00:00Z"), + () -> assertThat(contentCaptor.getValue()).contains("exit code 1") + ); + } + + @Test + void 상세_내용이_없으면_비어_있음을_표시해_전송한다() { + // given + DbBackupAlarmRequest request = new DbBackupAlarmRequest( + DbBackupAlarmType.BINLOG_UPLOAD_DELAYED, "i-detail-absent", OCCURRED_AT, null); + + // when + dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request); + + // then + verify(discordWebhookSender).send(anyString(), contains("상세: -"), anyList()); + } + } + + @Nested + @DisplayName("담당 역할 멘션") + class 담당_역할을_멘션한다 { + + @Test + void 메시지_앞에_역할_멘션을_붙이고_해당_역할만_멘션을_허용한다() { + // given + DbBackupAlarmRequest request = 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, "i-mention-target"); + + // when + dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request); + + // then + ArgumentCaptor contentCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor> roleIdsCaptor = ArgumentCaptor.forClass(List.class); + verify(discordWebhookSender).send(anyString(), contentCaptor.capture(), roleIdsCaptor.capture()); + assertAll( + () -> assertThat(contentCaptor.getValue()).startsWith("<@&%s>".formatted(MENTION_ROLE_ID)), + () -> assertThat(roleIdsCaptor.getValue()).containsExactly(MENTION_ROLE_ID) + ); + } + + @Test + void 멘션_문자열_뒤에_알림_본문이_이어진다() { + // given + DbBackupAlarmRequest request = 백업_알림_요청(DbBackupAlarmType.BINLOG_GAP_DETECTED, "i-mention-body-target"); + + // when + dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request); + + // then + ArgumentCaptor contentCaptor = ArgumentCaptor.forClass(String.class); + verify(discordWebhookSender).send(anyString(), contentCaptor.capture(), anyList()); + assertThat(contentCaptor.getValue()) + .contains("MySQL 백업 알림: " + DbBackupAlarmType.BINLOG_GAP_DETECTED.getDisplayName()); + } + } + + @Nested + @DisplayName("내부 호출자 인증") + class 내부_호출자를_인증한다 { + + @Test + void 토큰이_일치하지_않으면_예외_응답을_반환한다() { + // given + DbBackupAlarmRequest request = 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, INSTANCE_ID); + + // when, then + assertThatThrownBy(() -> dbBackupAlarmService.alarmBackupFailure("wrong-token", request)) + .isInstanceOf(CustomException.class); + verify(discordWebhookSender, never()).send(anyString(), anyString(), anyList()); + } + + @Test + void 토큰이_없으면_예외_응답을_반환한다() { + // given + DbBackupAlarmRequest request = 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, INSTANCE_ID); + + // when, then + assertThatThrownBy(() -> dbBackupAlarmService.alarmBackupFailure(null, request)) + .isInstanceOf(CustomException.class); + verify(discordWebhookSender, never()).send(anyString(), anyString(), anyList()); + } + } + + @Nested + @DisplayName("중복 알림 억제") + class 중복_알림을_억제한다 { + + @Test + void 같은_유형과_인스턴스로_반복_요청하면_한_번만_전송한다() { + // given + String instanceId = "i-duplicated-target"; + DbBackupAlarmRequest request = 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, instanceId); + + // when + dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request); + dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request); + + // then + verify(discordWebhookSender, times(1)).send(anyString(), contains(instanceId), anyList()); + } + + @Test + void 유형이_다르면_각각_전송한다() { + // given + String instanceId = "i-different-type-target"; + + // when + dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, instanceId)); + dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, 백업_알림_요청(DbBackupAlarmType.BINLOG_UPLOAD_FAILED, instanceId)); + + // then + assertAll( + () -> verify(discordWebhookSender) + .send(anyString(), contains(DbBackupAlarmType.DUMP_FAILED.getDisplayName()), anyList()), + () -> verify(discordWebhookSender) + .send(anyString(), contains(DbBackupAlarmType.BINLOG_UPLOAD_FAILED.getDisplayName()), anyList()) + ); + } + + @Test + void 인스턴스가_다르면_각각_전송한다() { + // given + String firstInstanceId = "i-first-target"; + String secondInstanceId = "i-second-target"; + + // when + dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, firstInstanceId)); + dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, secondInstanceId)); + + // then + assertAll( + () -> verify(discordWebhookSender, times(1)).send(anyString(), contains(firstInstanceId), anyList()), + () -> verify(discordWebhookSender, times(1)).send(anyString(), contains(secondInstanceId), anyList()) + ); + } + + @Test + void 전송에_실패하면_억제를_해제해_다음_요청을_다시_전송한다() { + // given + String instanceId = "i-send-failed-target"; + reset(discordWebhookSender); + when(discordWebhookSender.send(anyString(), anyString(), anyList())).thenReturn(false); + DbBackupAlarmRequest request = 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, instanceId); + + // when + dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request); + dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request); + + // then + verify(discordWebhookSender, times(2)).send(anyString(), contains(instanceId), anyList()); + } + } +} diff --git a/src/test/java/com/example/solidconnection/alarm/service/DbBackupAlarmServiceTokenNotConfiguredTest.java b/src/test/java/com/example/solidconnection/alarm/service/DbBackupAlarmServiceTokenNotConfiguredTest.java new file mode 100644 index 000000000..fac4ea870 --- /dev/null +++ b/src/test/java/com/example/solidconnection/alarm/service/DbBackupAlarmServiceTokenNotConfiguredTest.java @@ -0,0 +1,53 @@ +package com.example.solidconnection.alarm.service; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +import com.example.solidconnection.alarm.domain.DbBackupAlarmType; +import com.example.solidconnection.alarm.dto.DbBackupAlarmRequest; +import com.example.solidconnection.common.discord.DiscordWebhookSender; +import com.example.solidconnection.common.exception.CustomException; +import com.example.solidconnection.support.TestContainerSpringBootTest; +import java.time.Instant; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.bean.override.mockito.MockitoBean; + +@TestContainerSpringBootTest +@TestPropertySource(properties = { + "internal-alarm.token=", + "discord.db-backup-fail-alarm.webhook-url=https://discord.test/webhooks/db-backup" +}) +@DisplayName("DB 백업 알림 인증 토큰 미설정 테스트") +class DbBackupAlarmServiceTokenNotConfiguredTest { + + @Autowired + private DbBackupAlarmService dbBackupAlarmService; + + @MockitoBean + private DiscordWebhookSender discordWebhookSender; + + @Test + void 인증_토큰이_설정되지_않으면_어떤_토큰으로_요청해도_예외_응답을_반환한다() { + // given + DbBackupAlarmRequest request = new DbBackupAlarmRequest( + DbBackupAlarmType.DUMP_FAILED, "i-0c5d57927ef9ca426", Instant.parse("2026-08-17T03:00:00Z"), null); + + // when, then + assertAll( + () -> assertThatThrownBy(() -> dbBackupAlarmService.alarmBackupFailure("any-token", request)) + .isInstanceOf(CustomException.class), + () -> assertThatThrownBy(() -> dbBackupAlarmService.alarmBackupFailure("", request)) + .isInstanceOf(CustomException.class), + () -> assertThatThrownBy(() -> dbBackupAlarmService.alarmBackupFailure(null, request)) + .isInstanceOf(CustomException.class) + ); + verify(discordWebhookSender, never()).send(anyString(), anyString(), anyList()); + } +} diff --git a/src/test/java/com/example/solidconnection/common/discord/DiscordWebhookSenderTest.java b/src/test/java/com/example/solidconnection/common/discord/DiscordWebhookSenderTest.java new file mode 100644 index 000000000..e5c00f741 --- /dev/null +++ b/src/test/java/com/example/solidconnection/common/discord/DiscordWebhookSenderTest.java @@ -0,0 +1,130 @@ +package com.example.solidconnection.common.discord; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.http.HttpEntity; +import org.springframework.http.MediaType; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; + +@DisplayName("DiscordWebhookSender 테스트") +class DiscordWebhookSenderTest { + + private static final String WEBHOOK_URL = "https://discord.test/webhooks/channel"; + private static final String CONTENT = "백업 실패 알림"; + private static final String ROLE_ID = "1234567890"; + + private RestTemplate restTemplate; + private DiscordWebhookSender discordWebhookSender; + + @BeforeEach + void setUp() { + restTemplate = mock(RestTemplate.class); + discordWebhookSender = new DiscordWebhookSender(restTemplate); + } + + private HttpEntity> 전송된_요청() { + ArgumentCaptor>> requestCaptor = ArgumentCaptor.forClass(HttpEntity.class); + verify(restTemplate).postForEntity(eq(WEBHOOK_URL), requestCaptor.capture(), eq(Void.class)); + return requestCaptor.getValue(); + } + + @Nested + @DisplayName("메시지 전송") + class 메시지를_전송한다 { + + @Test + void 전달받은_webhook_url_로_content_를_전송한다() { + // when + boolean isSent = discordWebhookSender.send(WEBHOOK_URL, CONTENT); + + // then + HttpEntity> request = 전송된_요청(); + assertAll( + () -> assertThat(isSent).isTrue(), + () -> assertThat(request.getBody()).containsEntry("content", CONTENT), + () -> assertThat(request.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON) + ); + } + + @Test + void webhook_url_이_없으면_전송을_시도하지_않고_실패를_반환한다() { + // when + boolean nullUrlResult = discordWebhookSender.send(null, CONTENT); + boolean emptyUrlResult = discordWebhookSender.send("", CONTENT); + boolean blankUrlResult = discordWebhookSender.send(" ", CONTENT); + + // then + assertAll( + () -> assertThat(nullUrlResult).isFalse(), + () -> assertThat(emptyUrlResult).isFalse(), + () -> assertThat(blankUrlResult).isFalse(), + () -> verify(restTemplate, never()).postForEntity(anyString(), any(), eq(Void.class)) + ); + } + + @Test + void 전송이_실패하면_예외를_전파하지_않고_실패를_반환한다() { + // given + when(restTemplate.postForEntity(anyString(), any(), eq(Void.class))) + .thenThrow(new RestClientException("discord unavailable")); + + // when + boolean isSent = discordWebhookSender.send(WEBHOOK_URL, CONTENT); + + // then + assertThat(isSent).isFalse(); + } + } + + @Nested + @DisplayName("멘션 허용 범위") + class 멘션_허용_범위를_제한한다 { + + @Test + void 역할을_지정하지_않으면_모든_멘션을_차단한다() { + // when + discordWebhookSender.send(WEBHOOK_URL, CONTENT); + + // then + assertThat(전송된_요청().getBody()) + .containsEntry("allowed_mentions", Map.of("parse", List.of(), "roles", List.of())); + } + + @Test + void 지정한_역할만_멘션을_허용한다() { + // when + discordWebhookSender.send(WEBHOOK_URL, CONTENT, List.of(ROLE_ID)); + + // then + assertThat(전송된_요청().getBody()) + .containsEntry("allowed_mentions", Map.of("parse", List.of(), "roles", List.of(ROLE_ID))); + } + + @Test + void everyone_문자열이_섞여도_parse_가_비어_있어_채널_전체를_호출하지_않는다() { + // when + discordWebhookSender.send(WEBHOOK_URL, "@everyone 백업 실패", List.of(ROLE_ID)); + + // then + @SuppressWarnings("unchecked") + Map allowedMentions = (Map) 전송된_요청().getBody().get("allowed_mentions"); + assertThat(allowedMentions.get("parse")).isEqualTo(List.of()); + } + } +} From 57e2050e033d20138ad25cc5475a1c04281fa48e Mon Sep 17 00:00:00 2001 From: Hexeong <123macanic@naver.com> Date: Wed, 19 Aug 2026 23:26:04 +0900 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20=EB=B0=B1=EC=97=85=20=EC=95=8C?= =?UTF-8?q?=EB=A6=BC=20=EB=A6=AC=EB=B7=B0=20=EB=B0=98=EC=98=81=20=EB=B0=8F?= =?UTF-8?q?=20=EC=95=8C=EB=A6=BC=20=ED=94=BC=EB=A1=9C=EB=8F=84=20=EC=A0=95?= =?UTF-8?q?=EC=B1=85=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - webhook url 을 기동 시점에 검증하고, local 과 테스트 환경 설정을 추가한다 - 알림 유형별 심각도를 정의해 메시지에 표시한다 - webhook url 이 로그와 메트릭에 남지 않도록 전용 RestTemplate 을 사용한다 - 전송에 실패하면 502 를 반환해 호출자가 재시도할 수 있게 한다 - 같은 실패가 반복되면 억제 간격을 5분에서 6시간까지 점진적으로 늘린다 - 억제 상태를 원자적으로 갱신해 서버가 여러 대여도 한 번만 전송한다 Co-Authored-By: Claude Opus 5 --- .../alarm/config/DbBackupAlarmProperties.java | 10 + .../alarm/domain/DbBackupAlarmSeverity.java | 17 ++ .../alarm/domain/DbBackupAlarmType.java | 16 +- .../alarm/service/DbBackupAlarmService.java | 112 ++++++++--- .../config/client/RestTemplateConfig.java | 20 +- .../common/discord/DiscordWebhookSender.java | 14 +- .../common/exception/ErrorCode.java | 1 + .../service/DbBackupAlarmServiceTest.java | 188 +++++++++++++++--- src/test/resources/application.yml | 6 + 9 files changed, 322 insertions(+), 62 deletions(-) create mode 100644 src/main/java/com/example/solidconnection/alarm/domain/DbBackupAlarmSeverity.java diff --git a/src/main/java/com/example/solidconnection/alarm/config/DbBackupAlarmProperties.java b/src/main/java/com/example/solidconnection/alarm/config/DbBackupAlarmProperties.java index af6f8ac85..80d26a0a8 100644 --- a/src/main/java/com/example/solidconnection/alarm/config/DbBackupAlarmProperties.java +++ b/src/main/java/com/example/solidconnection/alarm/config/DbBackupAlarmProperties.java @@ -1,10 +1,20 @@ package com.example.solidconnection.alarm.config; +import jakarta.validation.constraints.NotBlank; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; +/* + * - webhookUrl 이 없으면 알림을 보낼 수 없으므로 기동 시점에 검증한다. + * - mentionRoleId 는 선택 값이며, 없으면 멘션 없이 알림만 보낸다. + * */ +@Validated @ConfigurationProperties(prefix = "discord.db-backup-fail-alarm") public record DbBackupAlarmProperties( + + @NotBlank String webhookUrl, + String mentionRoleId ) { diff --git a/src/main/java/com/example/solidconnection/alarm/domain/DbBackupAlarmSeverity.java b/src/main/java/com/example/solidconnection/alarm/domain/DbBackupAlarmSeverity.java new file mode 100644 index 000000000..b28fe641f --- /dev/null +++ b/src/main/java/com/example/solidconnection/alarm/domain/DbBackupAlarmSeverity.java @@ -0,0 +1,17 @@ +package com.example.solidconnection.alarm.domain; + +import lombok.Getter; + +@Getter +public enum DbBackupAlarmSeverity { + + WARNING("경고"), + CRITICAL("심각"), + ; + + private final String displayName; + + DbBackupAlarmSeverity(String displayName) { + this.displayName = displayName; + } +} diff --git a/src/main/java/com/example/solidconnection/alarm/domain/DbBackupAlarmType.java b/src/main/java/com/example/solidconnection/alarm/domain/DbBackupAlarmType.java index e5ad9bf42..1494a8f77 100644 --- a/src/main/java/com/example/solidconnection/alarm/domain/DbBackupAlarmType.java +++ b/src/main/java/com/example/solidconnection/alarm/domain/DbBackupAlarmType.java @@ -2,18 +2,24 @@ import lombok.Getter; +/* + * - 심각도는 호출자가 임의로 낮출 수 없도록 요청 값이 아니라 유형에서 결정한다. + * - 지연은 아직 복구 여지가 있어 경고로 두고, 기준점이나 복구 체인이 깨지는 경우는 심각으로 둔다. + * */ @Getter public enum DbBackupAlarmType { - DUMP_FAILED("전체 덤프 실패"), - BINLOG_UPLOAD_FAILED("바이너리 로그 업로드 실패"), - BINLOG_GAP_DETECTED("바이너리 로그 누락"), - BINLOG_UPLOAD_DELAYED("바이너리 로그 업로드 지연"), + DUMP_FAILED("전체 덤프 실패", DbBackupAlarmSeverity.CRITICAL), + BINLOG_UPLOAD_FAILED("바이너리 로그 업로드 실패", DbBackupAlarmSeverity.CRITICAL), + BINLOG_GAP_DETECTED("바이너리 로그 누락", DbBackupAlarmSeverity.CRITICAL), + BINLOG_UPLOAD_DELAYED("바이너리 로그 업로드 지연", DbBackupAlarmSeverity.WARNING), ; private final String displayName; + private final DbBackupAlarmSeverity severity; - DbBackupAlarmType(String displayName) { + DbBackupAlarmType(String displayName, DbBackupAlarmSeverity severity) { this.displayName = displayName; + this.severity = severity; } } diff --git a/src/main/java/com/example/solidconnection/alarm/service/DbBackupAlarmService.java b/src/main/java/com/example/solidconnection/alarm/service/DbBackupAlarmService.java index f6c41e473..c26ced580 100644 --- a/src/main/java/com/example/solidconnection/alarm/service/DbBackupAlarmService.java +++ b/src/main/java/com/example/solidconnection/alarm/service/DbBackupAlarmService.java @@ -1,5 +1,6 @@ package com.example.solidconnection.alarm.service; +import static com.example.solidconnection.common.exception.ErrorCode.DB_BACKUP_ALARM_SEND_FAILED; import static com.example.solidconnection.common.exception.ErrorCode.INTERNAL_ALARM_UNAUTHORIZED; import com.example.solidconnection.alarm.config.DbBackupAlarmProperties; @@ -18,16 +19,24 @@ import org.springframework.stereotype.Service; /* - * - DB EC2 는 private subnet 에 있어 Discord 로 직접 요청할 수 없다. - * - 따라서 백업 실패 이벤트를 전달받아 Discord 로 중계한다. + * - DB EC2 는 private subnet 에 있어 Discord 로 직접 요청할 수 없다. 따라서 백업 실패 이벤트를 전달받아 Discord 로 중계한다. + * - 같은 실패가 반복될 때 알림이 쌓이지 않도록 억제 간격을 점점 늘린다. + * - 억제 상태는 Redis 에 두고 원자적 연산으로 갱신하므로, 서버가 여러 대여도 한 대만 알림을 보낸다. * */ @Service @RequiredArgsConstructor @Slf4j public class DbBackupAlarmService { - private static final String SUPPRESSION_KEY_PREFIX = "db-backup-alarm:"; - private static final Duration SUPPRESSION_TTL = Duration.ofMinutes(10); + private static final String MUTE_KEY_PREFIX = "db-backup-alarm:mute:"; + private static final String COUNT_KEY_PREFIX = "db-backup-alarm:count:"; + private static final Duration COUNT_TTL = Duration.ofHours(12); + private static final List MUTE_DURATIONS = List.of( + Duration.ofMinutes(5), + Duration.ofMinutes(15), + Duration.ofHours(1), + Duration.ofHours(6) + ); private static final String ROLE_MENTION_FORMAT = "<@&%s>"; private static final String EMPTY_DETAIL = "-"; @@ -42,17 +51,25 @@ public class DbBackupAlarmService { public void alarmBackupFailure(String token, DbBackupAlarmRequest request) { validateToken(token); - String suppressionKey = buildSuppressionKey(request); - if (isSuppressed(suppressionKey)) { + String alarmKey = buildAlarmKey(request); + String muteKey = MUTE_KEY_PREFIX + alarmKey; + String countKey = COUNT_KEY_PREFIX + alarmKey; + + if (!acquireAlarmGate(muteKey)) { return; } + long alarmCount = increaseAlarmCount(countKey); + Duration muteDuration = resolveMuteDuration(alarmCount); + extendAlarmGate(muteKey, muteDuration); + boolean isSent = discordWebhookSender.send( dbBackupAlarmProperties.webhookUrl(), - buildMessage(request), + buildMessage(request, alarmCount, muteDuration), mentionableRoleIds() ); if (!isSent) { - releaseSuppression(suppressionKey); + releaseAlarmGate(muteKey, countKey); + throw new CustomException(DB_BACKUP_ALARM_SEND_FAILED); } } @@ -73,31 +90,70 @@ private void validateToken(String token) { } } - private String buildSuppressionKey(DbBackupAlarmRequest request) { - return SUPPRESSION_KEY_PREFIX + request.type().name() + ":" + request.instanceId(); + private String buildAlarmKey(DbBackupAlarmRequest request) { + return request.type().name() + ":" + request.instanceId(); } /* - * - 같은 유형과 인스턴스의 알림이 반복되면 일정 시간 동안 전송하지 않는다. - * - Redis 를 사용할 수 없을 때는 알림 누락을 막기 위해 억제하지 않는다. + * - setIfAbsent 는 원자적이므로 여러 서버가 동시에 요청받아도 한 대만 통과한다. + * - 통과하지 못하면 억제 중이거나 다른 서버가 방금 알림을 보낸 것이므로 전송하지 않는다. + * - Redis 를 사용할 수 없을 때는 알림 누락을 막기 위해 통과시킨다. * */ - private boolean isSuppressed(String suppressionKey) { + private boolean acquireAlarmGate(String muteKey) { try { - Boolean isFirstAlarm = redisTemplate.opsForValue().setIfAbsent(suppressionKey, "1", SUPPRESSION_TTL); - return !Boolean.TRUE.equals(isFirstAlarm); + Boolean isAcquired = redisTemplate.opsForValue() + .setIfAbsent(muteKey, "1", MUTE_DURATIONS.getFirst()); + return Boolean.TRUE.equals(isAcquired); } catch (Exception e) { - log.error("백업 알림 중복 억제 상태를 확인하지 못해 알림을 그대로 전송합니다. key={}", suppressionKey, e); - return false; + log.error("백업 알림 억제 상태를 확인하지 못해 알림을 그대로 전송합니다. key={}", muteKey, e); + return true; } } - private String buildMessage(DbBackupAlarmRequest request) { - return buildRoleMention() + "[%s] MySQL 백업 알림: %s\n인스턴스: %s\n발생 시각: %s\n상세: %s" + /* + * - 게이트를 통과한 요청만 카운트하므로 서버가 여러 대여도 연속 발생 횟수가 부풀지 않는다. + * */ + private long increaseAlarmCount(String countKey) { + try { + Long alarmCount = redisTemplate.opsForValue().increment(countKey); + redisTemplate.expire(countKey, COUNT_TTL); + if (alarmCount == null) { + return 1L; + } + return alarmCount; + } catch (Exception e) { + log.error("백업 알림 연속 발생 횟수를 증가하지 못했습니다. key={}", countKey, e); + return 1L; + } + } + + private Duration resolveMuteDuration(long alarmCount) { + int index = (int) Math.min(alarmCount, MUTE_DURATIONS.size()) - 1; + return MUTE_DURATIONS.get(Math.max(index, 0)); + } + + /* + * - 최초 잠금은 가장 짧은 간격으로 걸어두고, 연속 발생 횟수에 맞는 간격으로 늘린다. + * */ + private void extendAlarmGate(String muteKey, Duration muteDuration) { + try { + redisTemplate.expire(muteKey, muteDuration); + } catch (Exception e) { + log.error("백업 알림 억제 간격을 늘리지 못했습니다. key={}", muteKey, e); + } + } + + private String buildMessage(DbBackupAlarmRequest request, long alarmCount, Duration muteDuration) { + return buildRoleMention() + + "[%s] [%s] MySQL 백업 알림: %s\n인스턴스: %s\n발생 시각: %s\n연속 발생: %d회 (다음 %s 동안 같은 알림을 보내지 않습니다)\n상세: %s" .formatted( environment.toUpperCase(), + request.type().getSeverity().getDisplayName(), request.type().getDisplayName(), request.instanceId(), request.occurredAt(), + alarmCount, + formatDuration(muteDuration), resolveDetail(request.detail()) ); } @@ -113,6 +169,14 @@ private String buildRoleMention() { return ROLE_MENTION_FORMAT.formatted(mentionRoleId) + "\n"; } + private String formatDuration(Duration duration) { + long hours = duration.toHours(); + if (hours > 0) { + return hours + "시간"; + } + return duration.toMinutes() + "분"; + } + private String resolveDetail(String detail) { if (detail == null || detail.isBlank()) { return EMPTY_DETAIL; @@ -129,13 +193,15 @@ private List mentionableRoleIds() { } /* - * - 전송에 실패하면 억제 상태를 되돌려 다음 백업 주기의 알림이 막히지 않게 한다. + * - 전송에 실패하면 억제와 횟수를 되돌려 다음 요청이 다시 알림을 시도할 수 있게 한다. + * - dump 는 하루 한 번 실행되므로 실패를 그대로 두면 그날의 알림이 사라진다. * */ - private void releaseSuppression(String suppressionKey) { + private void releaseAlarmGate(String muteKey, String countKey) { try { - redisTemplate.delete(suppressionKey); + redisTemplate.delete(muteKey); + redisTemplate.opsForValue().decrement(countKey); } catch (Exception e) { - log.error("백업 알림 중복 억제 상태를 해제하지 못했습니다. key={}", suppressionKey, e); + log.error("백업 알림 억제 상태를 해제하지 못했습니다. key={}", muteKey, e); } } } diff --git a/src/main/java/com/example/solidconnection/common/config/client/RestTemplateConfig.java b/src/main/java/com/example/solidconnection/common/config/client/RestTemplateConfig.java index 87b43eb28..c93c6eacb 100644 --- a/src/main/java/com/example/solidconnection/common/config/client/RestTemplateConfig.java +++ b/src/main/java/com/example/solidconnection/common/config/client/RestTemplateConfig.java @@ -4,16 +4,32 @@ import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.http.client.SimpleClientHttpRequestFactory; import org.springframework.web.client.RestTemplate; @Configuration public class RestTemplateConfig { + private static final Duration TIMEOUT = Duration.ofSeconds(5); + @Bean public RestTemplate restTemplate(RestTemplateBuilder restTemplateBuilder) { return restTemplateBuilder - .setConnectTimeout(Duration.ofSeconds(5)) - .setReadTimeout(Duration.ofSeconds(5)) + .connectTimeout(TIMEOUT) + .readTimeout(TIMEOUT) .build(); } + + /* + * - Discord webhook url 은 경로에 인증 토큰을 포함한다. + * - RestTemplateBuilder 로 만든 RestTemplate 은 observation 이 적용되어 요청 url 이 메트릭 태그로 남을 수 있으므로, + * webhook 전송에는 observation 이 붙지 않는 별도 인스턴스를 사용한다. + * */ + @Bean + public RestTemplate discordWebhookRestTemplate() { + SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory(); + requestFactory.setConnectTimeout((int) TIMEOUT.toMillis()); + requestFactory.setReadTimeout((int) TIMEOUT.toMillis()); + return new RestTemplate(requestFactory); + } } diff --git a/src/main/java/com/example/solidconnection/common/discord/DiscordWebhookSender.java b/src/main/java/com/example/solidconnection/common/discord/DiscordWebhookSender.java index 0a7ca3dd5..eeb957e9b 100644 --- a/src/main/java/com/example/solidconnection/common/discord/DiscordWebhookSender.java +++ b/src/main/java/com/example/solidconnection/common/discord/DiscordWebhookSender.java @@ -8,19 +8,21 @@ import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.stereotype.Component; +import org.springframework.web.client.RestClientResponseException; import org.springframework.web.client.RestTemplate; /* * - Discord Webhook 으로 메시지를 전송한다. * - 알림 전송 실패가 호출한 기능을 실패시키지 않도록 예외를 격리하고, 전송 여부를 반환해 후속 처리를 맡긴다. * - 채널별로 webhook url 이 다르므로 url 을 인자로 받는다. + * - webhook url 은 경로에 인증 토큰을 포함하므로 로그와 메트릭에 남기지 않는다. * */ @Component @RequiredArgsConstructor @Slf4j public class DiscordWebhookSender { - private final RestTemplate restTemplate; + private final RestTemplate discordWebhookRestTemplate; public boolean send(String webhookUrl, String content) { return send(webhookUrl, content, List.of()); @@ -28,7 +30,7 @@ public boolean send(String webhookUrl, String content) { /* * - mentionableRoleIds 에 지정한 역할만 멘션할 수 있다. - * - @everyone 과 @here 는 항상 차단되므로 content 에 섞여 들어와도 채널 전체를 호출하지 않는다. + * - everyone 과 here 멘션은 항상 차단되므로 content 에 섞여 들어와도 채널 전체를 호출하지 않는다. * */ public boolean send(String webhookUrl, String content, List mentionableRoleIds) { if (webhookUrl == null || webhookUrl.isBlank()) { @@ -36,10 +38,14 @@ public boolean send(String webhookUrl, String content, List mentionableR return false; } try { - restTemplate.postForEntity(webhookUrl, buildRequest(content, mentionableRoleIds), Void.class); + discordWebhookRestTemplate.postForEntity(webhookUrl, buildRequest(content, mentionableRoleIds), Void.class); return true; + } catch (RestClientResponseException e) { + // 예외 메시지에 요청 url 이 포함되므로 상태 코드만 남긴다. + log.error("Discord 알림 전송이 실패 응답을 받았습니다. status={}", e.getStatusCode().value()); + return false; } catch (Exception e) { - log.error("Discord 알림 전송에 실패했습니다.", e); + log.error("Discord 알림 전송에 실패했습니다. exception={}", e.getClass().getSimpleName()); return false; } } diff --git a/src/main/java/com/example/solidconnection/common/exception/ErrorCode.java b/src/main/java/com/example/solidconnection/common/exception/ErrorCode.java index ebf0e0102..ce8621b8e 100644 --- a/src/main/java/com/example/solidconnection/common/exception/ErrorCode.java +++ b/src/main/java/com/example/solidconnection/common/exception/ErrorCode.java @@ -193,6 +193,7 @@ public enum ErrorCode { // internal alarm INTERNAL_ALARM_UNAUTHORIZED(HttpStatus.UNAUTHORIZED.value(), "요청을 인증할 수 없습니다."), + DB_BACKUP_ALARM_SEND_FAILED(HttpStatus.BAD_GATEWAY.value(), "알림 전송에 실패했습니다. 잠시 후 다시 시도해주세요."), // general JSON_PARSING_FAILED(HttpStatus.BAD_REQUEST.value(), "JSON 파싱을 할 수 없습니다."), diff --git a/src/test/java/com/example/solidconnection/alarm/service/DbBackupAlarmServiceTest.java b/src/test/java/com/example/solidconnection/alarm/service/DbBackupAlarmServiceTest.java index 0e432eb94..7ecacd135 100644 --- a/src/test/java/com/example/solidconnection/alarm/service/DbBackupAlarmServiceTest.java +++ b/src/test/java/com/example/solidconnection/alarm/service/DbBackupAlarmServiceTest.java @@ -19,12 +19,18 @@ import com.example.solidconnection.support.TestContainerSpringBootTest; import java.time.Instant; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.http.HttpStatus; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.bean.override.mockito.MockitoBean; @@ -41,11 +47,15 @@ class DbBackupAlarmServiceTest { private static final String WEBHOOK_URL = "https://discord.test/webhooks/db-backup"; private static final String MENTION_ROLE_ID = "1234567890"; private static final String INSTANCE_ID = "i-0c5d57927ef9ca426"; - private static final Instant OCCURRED_AT = Instant.parse("2026-08-17T03:00:00Z"); + private static final Instant OCCURRED_AT = Instant.parse("2026-08-18T03:00:00Z"); + private static final String MUTE_KEY_PREFIX = "db-backup-alarm:mute:"; @Autowired private DbBackupAlarmService dbBackupAlarmService; + @Autowired + private RedisTemplate redisTemplate; + @MockitoBean private DiscordWebhookSender discordWebhookSender; @@ -59,6 +69,10 @@ void setUp() { return new DbBackupAlarmRequest(type, instanceId, OCCURRED_AT, "exit code 1"); } + private String 억제_키(DbBackupAlarmType type, String instanceId) { + return MUTE_KEY_PREFIX + type.name() + ":" + instanceId; + } + @Nested @DisplayName("백업 실패 알림 전송") class 백업_실패_알림을_전송한다 { @@ -79,7 +93,7 @@ class 백업_실패_알림을_전송한다 { () -> assertThat(urlCaptor.getValue()).isEqualTo(WEBHOOK_URL), () -> assertThat(contentCaptor.getValue()).contains(DbBackupAlarmType.DUMP_FAILED.getDisplayName()), () -> assertThat(contentCaptor.getValue()).contains(INSTANCE_ID), - () -> assertThat(contentCaptor.getValue()).contains("2026-08-17T03:00:00Z"), + () -> assertThat(contentCaptor.getValue()).contains("2026-08-18T03:00:00Z"), () -> assertThat(contentCaptor.getValue()).contains("exit code 1") ); } @@ -98,6 +112,31 @@ class 백업_실패_알림을_전송한다 { } } + @Nested + @DisplayName("알림 심각도") + class 알림_심각도를_표시한다 { + + @Test + void 덤프_실패는_심각으로_표시한다() { + // when + dbBackupAlarmService.alarmBackupFailure( + VALID_TOKEN, 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, "i-severity-critical")); + + // then + verify(discordWebhookSender).send(anyString(), contains("[심각]"), anyList()); + } + + @Test + void 업로드_지연은_경고로_표시한다() { + // when + dbBackupAlarmService.alarmBackupFailure( + VALID_TOKEN, 백업_알림_요청(DbBackupAlarmType.BINLOG_UPLOAD_DELAYED, "i-severity-warning")); + + // then + verify(discordWebhookSender).send(anyString(), contains("[경고]"), anyList()); + } + } + @Nested @DisplayName("담당 역할 멘션") class 담당_역할을_멘션한다 { @@ -119,21 +158,6 @@ class 담당_역할을_멘션한다 { () -> assertThat(roleIdsCaptor.getValue()).containsExactly(MENTION_ROLE_ID) ); } - - @Test - void 멘션_문자열_뒤에_알림_본문이_이어진다() { - // given - DbBackupAlarmRequest request = 백업_알림_요청(DbBackupAlarmType.BINLOG_GAP_DETECTED, "i-mention-body-target"); - - // when - dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request); - - // then - ArgumentCaptor contentCaptor = ArgumentCaptor.forClass(String.class); - verify(discordWebhookSender).send(anyString(), contentCaptor.capture(), anyList()); - assertThat(contentCaptor.getValue()) - .contains("MySQL 백업 알림: " + DbBackupAlarmType.BINLOG_GAP_DETECTED.getDisplayName()); - } } @Nested @@ -164,8 +188,8 @@ class 내부_호출자를_인증한다 { } @Nested - @DisplayName("중복 알림 억제") - class 중복_알림을_억제한다 { + @DisplayName("알림 피로도 억제") + class 알림_피로도를_억제한다 { @Test void 같은_유형과_인스턴스로_반복_요청하면_한_번만_전송한다() { @@ -200,21 +224,106 @@ class 중복_알림을_억제한다 { } @Test - void 인스턴스가_다르면_각각_전송한다() { + void 연속_발생_횟수가_늘어나면_억제_간격도_늘어난다() { // given - String firstInstanceId = "i-first-target"; - String secondInstanceId = "i-second-target"; + String instanceId = "i-backoff-target"; + DbBackupAlarmRequest request = 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, instanceId); + String muteKey = 억제_키(DbBackupAlarmType.DUMP_FAILED, instanceId); + + // when - 억제를 강제로 풀어 다음 단계를 관찰한다 + for (int i = 0; i < 5; i++) { + dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request); + redisTemplate.delete(muteKey); + } + + // then + ArgumentCaptor contentCaptor = ArgumentCaptor.forClass(String.class); + verify(discordWebhookSender, times(5)).send(anyString(), contentCaptor.capture(), anyList()); + List contents = contentCaptor.getAllValues(); + assertAll( + () -> assertThat(contents.get(0)).contains("연속 발생: 1회").contains("다음 5분"), + () -> assertThat(contents.get(1)).contains("연속 발생: 2회").contains("다음 15분"), + () -> assertThat(contents.get(2)).contains("연속 발생: 3회").contains("다음 1시간"), + () -> assertThat(contents.get(3)).contains("연속 발생: 4회").contains("다음 6시간"), + () -> assertThat(contents.get(4)).contains("연속 발생: 5회").contains("다음 6시간") + ); + } + + @Test + void 억제_간격은_연속_발생_횟수에_맞게_남은_시간으로_설정된다() { + // given + String instanceId = "i-mute-ttl-target"; + DbBackupAlarmRequest request = 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, instanceId); + String muteKey = 억제_키(DbBackupAlarmType.DUMP_FAILED, instanceId); // when - dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, firstInstanceId)); - dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, secondInstanceId)); + dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request); + Long firstTtl = redisTemplate.getExpire(muteKey, TimeUnit.SECONDS); + redisTemplate.delete(muteKey); + dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request); + Long secondTtl = redisTemplate.getExpire(muteKey, TimeUnit.SECONDS); // then assertAll( - () -> verify(discordWebhookSender, times(1)).send(anyString(), contains(firstInstanceId), anyList()), - () -> verify(discordWebhookSender, times(1)).send(anyString(), contains(secondInstanceId), anyList()) + () -> assertThat(firstTtl).isBetween(1L, 300L), + () -> assertThat(secondTtl).isBetween(301L, 900L) ); } + } + + @Nested + @DisplayName("다중 서버 동시 요청") + class 다중_서버에서_동시에_요청해도_안전하다 { + + @Test + void 동시에_같은_알림이_들어오면_한_번만_전송한다() throws Exception { + // given + int threadCount = 10; + String instanceId = "i-concurrent-target"; + DbBackupAlarmRequest request = 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, instanceId); + ExecutorService executorService = Executors.newFixedThreadPool(threadCount); + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch endLatch = new CountDownLatch(threadCount); + + // when + for (int i = 0; i < threadCount; i++) { + executorService.submit(() -> { + try { + startLatch.await(); + dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request); + } catch (Exception e) { + // 억제된 요청과 인터럽트는 무시한다 + } finally { + endLatch.countDown(); + } + }); + } + startLatch.countDown(); + endLatch.await(10, TimeUnit.SECONDS); + executorService.shutdown(); + + // then + verify(discordWebhookSender, times(1)).send(anyString(), contains(instanceId), anyList()); + } + } + + @Nested + @DisplayName("전송 실패 처리") + class 전송_실패를_처리한다 { + + @Test + void 전송에_실패하면_502_응답을_반환한다() { + // given + reset(discordWebhookSender); + when(discordWebhookSender.send(anyString(), anyString(), anyList())).thenReturn(false); + DbBackupAlarmRequest request = 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, "i-send-failed-status"); + + // when, then + assertThatThrownBy(() -> dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request)) + .isInstanceOf(CustomException.class) + .extracting("code") + .isEqualTo(HttpStatus.BAD_GATEWAY.value()); + } @Test void 전송에_실패하면_억제를_해제해_다음_요청을_다시_전송한다() { @@ -225,11 +334,34 @@ class 중복_알림을_억제한다 { DbBackupAlarmRequest request = 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, instanceId); // when - dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request); - dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request); + assertThatThrownBy(() -> dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request)) + .isInstanceOf(CustomException.class); + assertThatThrownBy(() -> dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request)) + .isInstanceOf(CustomException.class); // then verify(discordWebhookSender, times(2)).send(anyString(), contains(instanceId), anyList()); } + + @Test + void 전송에_실패하면_연속_발생_횟수가_증가하지_않는다() { + // given + String instanceId = "i-count-rollback-target"; + String muteKey = 억제_키(DbBackupAlarmType.DUMP_FAILED, instanceId); + DbBackupAlarmRequest request = 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, instanceId); + reset(discordWebhookSender); + when(discordWebhookSender.send(anyString(), anyString(), anyList())).thenReturn(false); + assertThatThrownBy(() -> dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request)) + .isInstanceOf(CustomException.class); + + // when - 이번에는 전송이 성공한다 + reset(discordWebhookSender); + when(discordWebhookSender.send(anyString(), anyString(), anyList())).thenReturn(true); + redisTemplate.delete(muteKey); + dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request); + + // then - 실패한 시도가 횟수에 반영되지 않아 다시 1회차로 알린다 + verify(discordWebhookSender).send(anyString(), contains("연속 발생: 1회"), anyList()); + } } } diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml index bba4cedbf..55441f122 100644 --- a/src/test/resources/application.yml +++ b/src/test/resources/application.yml @@ -107,3 +107,9 @@ token: black-list: storage-key-prefix: "BLACKLIST" expire-time: 10m +discord: + webhook-url: "https://discord.test/webhooks/legacy" + db-backup-fail-alarm: + webhook-url: "https://discord.test/webhooks/db-backup" +internal-alarm: + token: "test-internal-alarm-token" From 6ab3a9fdf339c7055fb190472eeb5441ef2d3b63 Mon Sep 17 00:00:00 2001 From: Hexeong <123macanic@naver.com> Date: Wed, 19 Aug 2026 23:36:08 +0900 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20=EB=B0=B1=EC=97=85=20=EC=95=8C?= =?UTF-8?q?=EB=A6=BC=20=EC=96=B5=EC=A0=9C=20=EC=83=81=ED=83=9C=20=EB=90=98?= =?UTF-8?q?=EB=8F=8C=EB=A6=AC=EA=B8=B0=EB=A5=BC=20=EC=9B=90=EC=9E=90=20?= =?UTF-8?q?=EC=97=B0=EC=82=B0=EC=9C=BC=EB=A1=9C=20=EC=B2=98=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 억제 해제와 연속 발생 횟수 감소를 lua 로 묶어, 그 사이 다른 서버가 증가시킨 횟수를 줄이지 않게 한다 - 증가가 반영되지 않은 상태에서 되돌려도 횟수가 음수가 되지 않도록 하한을 둔다 Co-Authored-By: Claude Opus 5 --- .../alarm/service/DbBackupAlarmService.java | 23 ++++++++-- .../common/config/redis/RedisConfig.java | 6 +++ .../scripts/releaseDbBackupAlarm.lua | 13 ++++++ .../service/DbBackupAlarmServiceTest.java | 44 +++++++++++++++++++ 4 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 src/main/resources/scripts/releaseDbBackupAlarm.lua diff --git a/src/main/java/com/example/solidconnection/alarm/service/DbBackupAlarmService.java b/src/main/java/com/example/solidconnection/alarm/service/DbBackupAlarmService.java index c26ced580..8fb1a189c 100644 --- a/src/main/java/com/example/solidconnection/alarm/service/DbBackupAlarmService.java +++ b/src/main/java/com/example/solidconnection/alarm/service/DbBackupAlarmService.java @@ -12,10 +12,11 @@ import java.security.MessageDigest; import java.time.Duration; import java.util.List; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.core.script.RedisScript; import org.springframework.stereotype.Service; /* @@ -24,7 +25,6 @@ * - 억제 상태는 Redis 에 두고 원자적 연산으로 갱신하므로, 서버가 여러 대여도 한 대만 알림을 보낸다. * */ @Service -@RequiredArgsConstructor @Slf4j public class DbBackupAlarmService { @@ -44,10 +44,25 @@ public class DbBackupAlarmService { private final DbBackupAlarmProperties dbBackupAlarmProperties; private final InternalAlarmAuthProperties internalAlarmAuthProperties; private final RedisTemplate redisTemplate; + private final RedisScript releaseDbBackupAlarmLuaScript; @Value("${spring.profiles.active:}") private String environment; + public DbBackupAlarmService( + DiscordWebhookSender discordWebhookSender, + DbBackupAlarmProperties dbBackupAlarmProperties, + InternalAlarmAuthProperties internalAlarmAuthProperties, + RedisTemplate redisTemplate, + @Qualifier("releaseDbBackupAlarmScript") RedisScript releaseDbBackupAlarmLuaScript + ) { + this.discordWebhookSender = discordWebhookSender; + this.dbBackupAlarmProperties = dbBackupAlarmProperties; + this.internalAlarmAuthProperties = internalAlarmAuthProperties; + this.redisTemplate = redisTemplate; + this.releaseDbBackupAlarmLuaScript = releaseDbBackupAlarmLuaScript; + } + public void alarmBackupFailure(String token, DbBackupAlarmRequest request) { validateToken(token); @@ -195,11 +210,11 @@ private List mentionableRoleIds() { /* * - 전송에 실패하면 억제와 횟수를 되돌려 다음 요청이 다시 알림을 시도할 수 있게 한다. * - dump 는 하루 한 번 실행되므로 실패를 그대로 두면 그날의 알림이 사라진다. + * - 억제 해제와 횟수 감소를 나누어 실행하면 그 사이에 다른 서버가 증가시킨 횟수를 잘못 줄이므로 lua 로 함께 처리한다. * */ private void releaseAlarmGate(String muteKey, String countKey) { try { - redisTemplate.delete(muteKey); - redisTemplate.opsForValue().decrement(countKey); + redisTemplate.execute(releaseDbBackupAlarmLuaScript, List.of(muteKey, countKey)); } catch (Exception e) { log.error("백업 알림 억제 상태를 해제하지 못했습니다. key={}", muteKey, e); } diff --git a/src/main/java/com/example/solidconnection/common/config/redis/RedisConfig.java b/src/main/java/com/example/solidconnection/common/config/redis/RedisConfig.java index f3041b02d..2b8f925a9 100644 --- a/src/main/java/com/example/solidconnection/common/config/redis/RedisConfig.java +++ b/src/main/java/com/example/solidconnection/common/config/redis/RedisConfig.java @@ -69,4 +69,10 @@ public RedisScript incrViewCountLuaScript() { Resource scriptSource = new ClassPathResource("scripts/incrViewCount.lua"); return RedisScript.of(scriptSource, Long.class); } + + @Bean(name = "releaseDbBackupAlarmScript") + public RedisScript releaseDbBackupAlarmLuaScript() { + Resource scriptSource = new ClassPathResource("scripts/releaseDbBackupAlarm.lua"); + return RedisScript.of(scriptSource, Long.class); + } } diff --git a/src/main/resources/scripts/releaseDbBackupAlarm.lua b/src/main/resources/scripts/releaseDbBackupAlarm.lua new file mode 100644 index 000000000..9b62368c0 --- /dev/null +++ b/src/main/resources/scripts/releaseDbBackupAlarm.lua @@ -0,0 +1,13 @@ +-- 알림 억제와 연속 발생 횟수를 하나의 원자 연산으로 되돌린다. +-- 두 명령을 나누어 실행하면 그 사이에 다른 서버가 게이트를 얻어 증가시킨 횟수를 잘못 줄일 수 있다. +-- KEYS[1]: 억제 키, KEYS[2]: 연속 발생 횟수 키 +redis.call('DEL', KEYS[1]) + +local count = tonumber(redis.call('GET', KEYS[2])) +-- 증가가 반영되지 않은 상태에서 줄이면 음수가 되므로, 1 이하이면 키를 지워 다음 알림이 1회차부터 시작하게 한다. +if count == nil or count <= 1 then + redis.call('DEL', KEYS[2]) + return 0 +end + +return redis.call('DECR', KEYS[2]) diff --git a/src/test/java/com/example/solidconnection/alarm/service/DbBackupAlarmServiceTest.java b/src/test/java/com/example/solidconnection/alarm/service/DbBackupAlarmServiceTest.java index 7ecacd135..0be09e412 100644 --- a/src/test/java/com/example/solidconnection/alarm/service/DbBackupAlarmServiceTest.java +++ b/src/test/java/com/example/solidconnection/alarm/service/DbBackupAlarmServiceTest.java @@ -29,7 +29,9 @@ import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.core.script.RedisScript; import org.springframework.http.HttpStatus; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.bean.override.mockito.MockitoBean; @@ -49,6 +51,7 @@ class DbBackupAlarmServiceTest { private static final String INSTANCE_ID = "i-0c5d57927ef9ca426"; private static final Instant OCCURRED_AT = Instant.parse("2026-08-18T03:00:00Z"); private static final String MUTE_KEY_PREFIX = "db-backup-alarm:mute:"; + private static final String COUNT_KEY_PREFIX = "db-backup-alarm:count:"; @Autowired private DbBackupAlarmService dbBackupAlarmService; @@ -56,6 +59,10 @@ class DbBackupAlarmServiceTest { @Autowired private RedisTemplate redisTemplate; + @Autowired + @Qualifier("releaseDbBackupAlarmScript") + private RedisScript releaseDbBackupAlarmLuaScript; + @MockitoBean private DiscordWebhookSender discordWebhookSender; @@ -73,6 +80,10 @@ void setUp() { return MUTE_KEY_PREFIX + type.name() + ":" + instanceId; } + private String 발생_횟수_키(DbBackupAlarmType type, String instanceId) { + return COUNT_KEY_PREFIX + type.name() + ":" + instanceId; + } + @Nested @DisplayName("백업 실패 알림 전송") class 백업_실패_알림을_전송한다 { @@ -343,6 +354,39 @@ class 전송_실패를_처리한다 { verify(discordWebhookSender, times(2)).send(anyString(), contains(instanceId), anyList()); } + @Test + void 전송에_실패하면_연속_발생_횟수_키를_남기지_않아_음수가_되지_않는다() { + // given + String instanceId = "i-count-floor-target"; + String countKey = 발생_횟수_키(DbBackupAlarmType.DUMP_FAILED, instanceId); + reset(discordWebhookSender); + when(discordWebhookSender.send(anyString(), anyString(), anyList())).thenReturn(false); + DbBackupAlarmRequest request = 백업_알림_요청(DbBackupAlarmType.DUMP_FAILED, instanceId); + + // when + assertThatThrownBy(() -> dbBackupAlarmService.alarmBackupFailure(VALID_TOKEN, request)) + .isInstanceOf(CustomException.class); + + // then + assertThat(redisTemplate.opsForValue().get(countKey)).isNull(); + } + + @Test + void 연속_발생_횟수가_없는_상태에서_되돌려도_음수가_되지_않는다() { + // given + String muteKey = 억제_키(DbBackupAlarmType.DUMP_FAILED, "i-script-floor-target"); + String countKey = 발생_횟수_키(DbBackupAlarmType.DUMP_FAILED, "i-script-floor-target"); + + // when - 증가가 반영되지 않은 상태를 재현한다 + Long remainingCount = redisTemplate.execute(releaseDbBackupAlarmLuaScript, List.of(muteKey, countKey)); + + // then + assertAll( + () -> assertThat(remainingCount).isZero(), + () -> assertThat(redisTemplate.opsForValue().get(countKey)).isNull() + ); + } + @Test void 전송에_실패하면_연속_발생_횟수가_증가하지_않는다() { // given