Skip to content

Instantly share code, notes, and snippets.

@kimyongin
Last active November 9, 2021 06:49
Show Gist options
  • Save kimyongin/d774b28e66fbbb611ca8b07329434caf to your computer and use it in GitHub Desktop.
Save kimyongin/d774b28e66fbbb611ca8b07329434caf to your computer and use it in GitHub Desktop.
package subscription.refresh.manager.task;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomStringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import slack.sender.service.SlackService;
import stamp.common.utils.DateTimeUtil;
import stamp.dao.dynamo.m.constants.ComparisonOperator;
import stamp.dao.dynamo.m.constants.TableColumns;
import stamp.dao.dynamo.m.constants.TableNames;
import stamp.dao.dynamo.m.entity.SubscriptionEntity;
import stamp.dao.dynamo.m.repository.SubscriptionRepository;
import stamp.dao.dynamo.m.service.ShardDaemonService;
import stamp.dao.dynamo.m.util.QueryScanExpressionBuilder;
import subscription.refresh.manager.service.SubscriptionRefreshService;
import javax.annotation.PostConstruct;
import java.text.MessageFormat;
import java.util.List;
import static stamp.dao.dynamo.m.util.QueryScanExpressionBuilder.OrderBy.Ascending;
@Slf4j
@Component
@RequiredArgsConstructor
public class RefreshByManualTask {
@Value("${shard-service-id.subscription-refresh}")
private String serviceId;
private final String daemonId = RandomStringUtils.randomAlphanumeric(10);
private final ShardDaemonService shardDaemonService;
private final SubscriptionRepository subscriptionRepository;
private final SubscriptionRefreshService subscriptionRefreshService;
private final SlackService slackService;
/**
schedule.health-check-ms=60000
schedule.adjust-ms=60000
schedule.consume-ms=60000
retire-shard-delete-ms=60000
shard-query-max-size=100
schedule.pulling-message=1000
manage-shard-count=1
**/
@Value("${schedule.health-check-ms}")
private long healthCheckIntervalMs;
@Value("${schedule.consume-ms}")
private long consumeIntervalMs;
@Value("${retire-shard-delete-ms}")
private long retireShardDeleteMs;
@Value("${shard-query-max-size}")
private int shardQueryMaxSize;
@Value("${manage-shard-count}")
private int manageShardCount;
@PostConstruct
public void postConstruct() {
shardDaemonService.initialize(serviceId, daemonId, this::hasShardData);
}
@Scheduled(fixedRateString = "${schedule.health-check-ms}")
protected void onHealthCheck() {
shardDaemonService.healthCheck();
}
@Scheduled(fixedDelayString = "${schedule.adjust-ms}")
protected void onAdjust() {
int myShardCount = shardDaemonService.adjust(healthCheckIntervalMs * 3, consumeIntervalMs * 3, retireShardDeleteMs);
if (myShardCount > manageShardCount) {
slackService.sendMessage(MessageFormat.format("{0} - {1} has {2} shard.", this.serviceId, this.daemonId, myShardCount));
}
}
@Scheduled(fixedDelayString = "${schedule.consume-ms}")
protected void onConsume() {
List<String> shardIds = shardDaemonService.getOwnedShardIds();
shardIds.forEach(shardDaemonService::updateLastConsumed);
shardIds.forEach(shardId -> {
List<SubscriptionEntity> subscriptionEntities = querySubscriptionToRefresh(shardId, shardQueryMaxSize);
subscriptionEntities.forEach(subscriptionRefreshService::handleScheduledCheck);
});
}
private boolean hasShardData(String shardId) {
QueryScanExpressionBuilder queryExpression = QueryScanExpressionBuilder.builder()
.indexName(TableNames.SUBSCRIPTION_GSI_SCHEDULE_GROUP_AND_SCHEDULE_DATE_MS)
.hashKey(TableColumns.SCHEDULE_GROUP, shardId)
.limit(1);
return subscriptionRepository.query(queryExpression).size() > 0;
}
private List<SubscriptionEntity> querySubscriptionToRefresh(String shardId, int size) {
QueryScanExpressionBuilder queryExpression = QueryScanExpressionBuilder.builder()
.indexName(TableNames.SUBSCRIPTION_GSI_SCHEDULE_GROUP_AND_SCHEDULE_DATE_MS)
.hashKey(TableColumns.SCHEDULE_GROUP, shardId)
.rangeKeyCompare(TableColumns.SCHEDULE_DATE_MS, ComparisonOperator.LT, DateTimeUtil.nowUtc0LongMs())
.limit(size)
.rangeOrderBy(Ascending);
return subscriptionRepository.query(queryExpression);
}
private void handleFail(Exception ex, SubscriptionEntity subscription) {
log.info("[REFRESH_TASK] id - {}, key - {}, handle fail - {}", subscription.getSubscriptionId(), subscription.getSubscriptionKey(), ex.toString());
}
}
package stamp.dao.dynamo.m.service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import stamp.common.utils.DateTimeUtil;
import stamp.dao.dynamo.m.constants.TableColumns;
import stamp.dao.dynamo.m.entity.DaemonEntity;
import stamp.dao.dynamo.m.entity.ShardEntity;
import stamp.dao.dynamo.m.repository.DaemonRepository;
import stamp.dao.dynamo.m.repository.ShardRepository;
import stamp.dao.dynamo.m.util.QueryScanExpressionBuilder;
import stamp.dao.dynamo.m.util.SaveExpressionBuilder;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
@Slf4j
@Service
@RequiredArgsConstructor
public class ShardDaemonService {
private final ShardRepository stampShardRepository;
private final DaemonRepository stampDaemonRepository;
private DaemonEntity me;
private List<ShardEntity> myShards = new ArrayList<>();
private Function<String, Boolean> hasShardData;
public void initialize(String serviceId, String daemonId, Function<String, Boolean> hasShardData) {
SaveExpressionBuilder saveExpressionBuilder = SaveExpressionBuilder.builder().notExists(TableColumns.DAEMON_ID);
me = DaemonEntity.builder()
.serviceId(serviceId)
.daemonId(daemonId)
.lastHealthCheckMs((DateTimeUtil.nowUtc0LongMs()))
.build();
stampDaemonRepository.save(me, saveExpressionBuilder);
this.hasShardData = hasShardData;
}
public void healthCheck() {
me.setLastHealthCheckMs(DateTimeUtil.nowUtc0LongMs());
stampDaemonRepository.save(me);
log.info(MessageFormat.format("[{0}][{1}] healthCheck", me.getServiceId(), me.getDaemonId()));
}
public int adjust(long healthCheckMaxDelayMs, long consumeMaxDelayMs, long retireShardDeleteMs) {
long now = DateTimeUtil.nowUtc0LongMs();
QueryScanExpressionBuilder queryByServiceId = QueryScanExpressionBuilder.builder().hashKey(TableColumns.SERVICE_ID, me.getServiceId());
List<DaemonEntity> daemons = stampDaemonRepository.query(queryByServiceId);
List<ShardEntity> shards = stampShardRepository.query(queryByServiceId);
// Daemon 분류 : Healthy, Unhealthy
List<DaemonEntity> healthyDaemons = new ArrayList<>();
List<DaemonEntity> unhealthyDaemons = new ArrayList<>();
daemons.forEach(daemon -> {
if (now - daemon.getLastHealthCheckMs() < healthCheckMaxDelayMs) {
healthyDaemons.add(daemon);
} else {
unhealthyDaemons.add(daemon);
}
});
// Delete Unhealthy Daemon
unhealthyDaemons.forEach(daemon -> deleteDaemon(daemon, "unhealthy"));
myShards = new ArrayList<>();
if (shards.size() == 0) {
return -1; // 처리할 샤드가 없다.
}
// Shard 분류 : MyShard, OtherShard, FreeShard
List<ShardEntity> freeShards = new ArrayList<>();
List<ShardEntity> otherShards = new ArrayList<>();
List<ShardEntity> retireShards = new ArrayList<>();
shards.forEach(shard -> {
String daemonId = shard.getOwnerDaemonId();
if (daemonId == null) {
freeShards.add(shard);
} else if (Objects.equals(me.getDaemonId(), daemonId)) {
myShards.add(shard);
} else if (!Objects.equals(me.getDaemonId(), daemonId)) {
otherShards.add(shard);
}
if (shard.getRetiredDateMs() != null) {
retireShards.add(shard);
}
});
// Release Delay Shard
otherShards.forEach(otherShard -> {
if (now - otherShard.getLastConsumedDateMs() > consumeMaxDelayMs) {
DaemonEntity daemon = stampDaemonRepository.load(otherShard.getServiceId(), otherShard.getOwnerDaemonId());
if (daemon == null || now - daemon.getLastHealthCheckMs() > healthCheckMaxDelayMs) {
releaseShard(otherShard, "delayed");
}
}
});
// Delete Retire Shard
retireShards.forEach(retireShard -> {
if (now - retireShard.getRetiredDateMs() > retireShardDeleteMs && !hasShardData.apply(retireShard.getShardId())) {
deleteShard(retireShard, "retired");
}
});
// Re balance(Acquire or Release) Shard
float myShardCount = myShards.size();
float idealShardCount = (float) shards.size() / (float) healthyDaemons.size();
float balance = myShardCount - idealShardCount;
if (balance >= 1) {
List<ShardEntity> releaseShards = myShards.stream()
.peek(shard -> releaseShard(shard, "exceed balance"))
.limit((long) balance)
.collect(Collectors.toList());
myShards.removeAll(releaseShards);
log.info(MessageFormat.format("[{0}][{1}] releaseCount : {2}", me.getServiceId(), me.getDaemonId(), releaseShards.size()));
} else if (balance < 0 && freeShards.size() > 0) {
List<ShardEntity> acquireShards = freeShards.stream()
.peek(shard -> acquireShard(shard, "under balance"))
.limit((long) Math.ceil(Math.abs(balance)))
.collect(Collectors.toList());
myShards.addAll(acquireShards);
log.info(MessageFormat.format("[{0}][{1}] acquireCount : {2}", me.getServiceId(), me.getDaemonId(), acquireShards.size()));
}
return CollectionUtils.isEmpty(myShards) ? 0 : myShards.size();
}
public List<String> getOwnedShardIds() {
return myShards.stream()
.map(ShardEntity::getShardId)
.collect(Collectors.toList());
}
public void updateLastConsumed(String shardId) {
Optional<ShardEntity> shardEntityOpt = myShards.stream().filter(entity -> Objects.equals(entity.getShardId(), shardId)).findFirst();
if (shardEntityOpt.isPresent()) {
ShardEntity stampShardEntity = shardEntityOpt.get();
stampShardEntity.setLastConsumedDateMs(DateTimeUtil.nowUtc0LongMs());
stampShardRepository.save(stampShardEntity);
}
}
private void acquireShard(ShardEntity stampShardEntity, String reason) {
SaveExpressionBuilder saveCondition = SaveExpressionBuilder.builder().notExists(TableColumns.OWNER_DAEMON_ID);
stampShardEntity.setOwnerDaemonId(me.getDaemonId());
stampShardEntity.setLastConsumedDateMs(DateTimeUtil.nowUtc0LongMs());
stampShardRepository.save(stampShardEntity, saveCondition);
log.info(MessageFormat.format("[{0}][{1}] acquireShard {2} because {3}", me.getServiceId(), me.getDaemonId(), stampShardEntity.getShardId(), reason));
}
private void releaseShard(ShardEntity stampShardEntity, String reason) {
SaveExpressionBuilder saveCondition = SaveExpressionBuilder.builder().eq(TableColumns.OWNER_DAEMON_ID, stampShardEntity.getOwnerDaemonId());
stampShardEntity.setOwnerDaemonId(null);
stampShardRepository.save(stampShardEntity, saveCondition);
log.info(MessageFormat.format("[{0}][{1}] releaseShard {2} because {3}", me.getServiceId(), me.getDaemonId(), stampShardEntity.getShardId(), reason));
}
private void deleteDaemon(DaemonEntity stampDaemonEntity, String reason) {
stampDaemonRepository.delete(stampDaemonEntity);
log.info(MessageFormat.format("[{0}][{1}] deleteDaemon because {2}", me.getServiceId(), me.getDaemonId(), reason));
}
private void deleteShard(ShardEntity stampShardEntity, String reason) {
stampShardRepository.delete(stampShardEntity);
log.info(MessageFormat.format("[{0}][{1}] deleteShard {2} because {3}", me.getServiceId(), me.getDaemonId(), stampShardEntity.getShardId(), reason));
}
}
@kimyongin
Copy link
Author

kimyongin commented Sep 18, 2021

[시간1. 데몬2개, 샤드2개가 처음으로 만들어진 상태]
--> 데몬1,2) 샤드1:무소유, 샤드2:무소유
1) 데몬1 밸런스 : 0 - 1 = -1 && 무소유샤드개수(2) > 0 --> abs(밸런스) 만큼 샤드 획득
2) 데몬2 밸런스 : 0 - 1 = -1 && 무소유샤드개수(2) > 0 --> abs(밸런스) 만큼 샤드 획득
--> 데몬1,2) 샤드1:데몬1, 샤드2:데몬2
※ 만약 동시에 같은 샤드를 획득 하려고 하면, 조건부 쓰기에 의해 둘중 하나는 획득을 실패하여 하나의 샤드만 소유된다.

[시간2. 데몬2가 언헬시에 빠진 상태]
--> 데몬1) 샤드1:데몬1, 샤드2:데몬2(언헬시)
1) 데몬1 에서 데몬2를 언헬시로 판단하고 삭제 처리한다
--> 데몬1) 샤드1:데몬1, 샤드2:무소유
2) 데몬1 밸런스 : 1 - 2 = -1 && 무소유샤드개수(1) > 0 --> abs(밸런스) 만큼 샤드 획득
--> 데몬1) 샤드1:데몬1, 샤드2:데몬1

[시간3. 데몬1만 구동중인 상태]
--> 데몬1) 샤드1:데몬1, 샤드2:데몬1
1) 데몬1 밸런스 : 2 - 2 = 0 --> no operation

[시간4. 데몬2가 언헬시에서 헬시로 다시 돌아온 상태]
--> 데몬1,2) 샤드1:데몬1, 샤드2:데몬1
1) 데몬2 밸런스 : 0 - 1 = -1 && 무소유샤드개수(0) > 0 --> no operation
2) 데몬1 밸런스 : 2 - 1 = +1 --> 밸런스 만큼 샤드 해제
--> 데몬1,2) 샤드1:데몬1, 샤드2:무소유
3) 데몬2 밸런스 : 0 - 1 = -1 && 무소유샤드개수(1) > 0 --> abs(밸런스) 만큼 샤드 획득
4) 데몬1 밸런스 : 1 - 1 = +0 --> no operation
--> 데몬1,2) 샤드1:데몬1, 샤드2:데몬2

[시간5. 샤드3 추가]
-->데몬1,2) 샤드1:데몬1, 샤드2:데몬2, 샤드3:무소유
1) 데몬1 밸런스 : 1 - 1.5 = -0.5 && 무소유샤드개수(1) > 0 --> ceil(abs(밸런스)) 만큼 샤드 획득
-->데몬1,2) 샤드1:데몬1, 샤드2:데몬2, 샤드3:데몬1
2) 데몬2 밸런스 : 1 - 1.5 = -0.5 && 무소유샤드개수(0) > 0 --> no operation
3) 데몬1 밸런스 : 2 - 1.5 = +0.5 --> no operation
-->데몬1,2) 샤드1:데몬1, 샤드2:데몬2, 샤드3:데몬1

@kimyongin
Copy link
Author

kimyongin commented Sep 18, 2021

현재 상황

plex 로그에는 2개 데몬이 health check 로그만 계속 찍는다. 다른 로그는 없다.

ddb daemon list

subscription-refresh	TEuPKbATIx	1631959482954
subscription-refresh	YEEjJO066v	1631959485924

ddb shard list

subscription-refresh	1	1631871087563	
subscription-refresh	2	1631959490260	YEEjJO066v

ddb 중요사항

모든 데몬은 healthy 하다
1번 샤드의 마지막 소비 시간이 9월 17일 금요일 오후 6:31(1631871087563)  이다

@kimyongin
Copy link
Author

kimyongin commented Sep 18, 2021

Sep 15, 2021 @ 18:29:46.035 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:29:42.962 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:28:45.973 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:28:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:27:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:27:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:26:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:26:43.076 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:25:45.934 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:25:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:24:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:24:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:23:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:23:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:22:45.934 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:22:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:21:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:21:42.962 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:20:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:20:42.962 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:19:45.938 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:19:43.004 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:18:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:18:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:17:46.076 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:17:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:16:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:16:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:15:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:15:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:15:15.845 [subscription-refresh][TEuPKbATIx] releaseCount : 1
Sep 15, 2021 @ 18:15:15.845 [subscription-refresh][TEuPKbATIx] releaseShard 1 because exceed balance
Sep 15, 2021 @ 18:14:46.146 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:14:46.140 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:14:46.133 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:14:46.127 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:14:46.120 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:14:46.114 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:14:46.108 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:14:46.102 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:14:46.095 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:14:46.089 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:14:46.082 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:14:46.074 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:14:46.067 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:14:46.059 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:14:46.053 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:14:46.046 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:14:46.040 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:14:46.011 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 18:14:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:13:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:12:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:11:42.962 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:10:43.087 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:09:42.962 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:08:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:07:42.962 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:06:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:05:42.962 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:04:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:03:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:02:43.167 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:02:08.066 [subscription-refresh][TEuPKbATIx] acquireShard 2 because under balance
Sep 15, 2021 @ 18:02:08.066 [subscription-refresh][TEuPKbATIx] acquireCount : 1
Sep 15, 2021 @ 18:01:42.962 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:01:08.045 [subscription-refresh][TEuPKbATIx] releaseShard 2 because delayed
Sep 15, 2021 @ 18:00:42.962 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 18:00:06.375 [subscription-refresh][TEuPKbATIx] deleteDaemon because unhealthy
Sep 15, 2021 @ 17:59:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:58:42.962 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:57:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:56:46.039 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:56:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:55:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:55:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:54:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:54:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:53:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:53:43.114 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:52:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:52:42.962 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:51:46.006 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:51:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:50:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:50:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:49:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:49:42.962 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:48:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:48:42.962 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:47:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:47:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:46:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:46:42.963 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:45:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:45:42.962 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:44:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:44:43.078 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:43:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:43:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:42:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:42:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:41:46.018 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:41:43.131 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:40:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:40:43.012 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:39:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:39:42.962 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:38:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:38:42.995 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:37:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:37:42.962 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:36:46.118 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:36:42.962 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:35:46.029 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:35:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:34:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:34:43.170 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:33:45.933 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:33:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:32:46.036 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:32:43.107 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:31:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:31:42.961 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 15, 2021 @ 17:30:45.935 [subscription-refresh][YEEjJO066v] healthCheck
Sep 15, 2021 @ 17:30:42.961 [subscription-refresh][TEuPKbATIx] healthCheck

@kimyongin
Copy link
Author

Sep 17, 2021 @ 18:35:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:34:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:34:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:33:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:33:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:32:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:32:42.977 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:31:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:31:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:31:30.854 [subscription-refresh][TEuPKbATIx] releaseCount : 1
Sep 17, 2021 @ 18:31:30.853 [subscription-refresh][TEuPKbATIx] releaseShard 1 because exceed balance
Sep 17, 2021 @ 18:30:45.933 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:30:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:29:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:29:43.056 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:28:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:28:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:28:30.131 [subscription-refresh][TEuPKbATIx] acquireCount : 1
Sep 17, 2021 @ 18:28:30.130 [subscription-refresh][TEuPKbATIx] acquireShard 1 because under balance
Sep 17, 2021 @ 18:27:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:27:43.199 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:26:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:26:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:26:30.089 [subscription-refresh][TEuPKbATIx] releaseCount : 1
Sep 17, 2021 @ 18:26:30.089 [subscription-refresh][TEuPKbATIx] releaseShard 1 because exceed balance
Sep 17, 2021 @ 18:25:46.014 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:25:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:24:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:24:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:23:46.065 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:23:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:22:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:22:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:21:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:21:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:20:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:20:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:20:28.285 [subscription-refresh][TEuPKbATIx] acquireCount : 1
Sep 17, 2021 @ 18:20:28.285 [subscription-refresh][TEuPKbATIx] acquireShard 1 because under balance
Sep 17, 2021 @ 18:19:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:19:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:18:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:18:43.105 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:17:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:17:43.005 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:17:28.065 [subscription-refresh][TEuPKbATIx] releaseCount : 1
Sep 17, 2021 @ 18:17:28.065 [subscription-refresh][TEuPKbATIx] releaseShard 1 because exceed balance
Sep 17, 2021 @ 18:16:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:16:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:16:27.818 [subscription-refresh][TEuPKbATIx] acquireShard 1 because under balance
Sep 17, 2021 @ 18:16:27.818 [subscription-refresh][TEuPKbATIx] acquireCount : 1
Sep 17, 2021 @ 18:15:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:15:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:15:27.791 [subscription-refresh][TEuPKbATIx] releaseShard 1 because exceed balance
Sep 17, 2021 @ 18:15:27.791 [subscription-refresh][TEuPKbATIx] releaseCount : 1
Sep 17, 2021 @ 18:14:46.055 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:14:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:14:27.545 [subscription-refresh][TEuPKbATIx] acquireShard 1 because under balance
Sep 17, 2021 @ 18:14:27.545 [subscription-refresh][TEuPKbATIx] acquireCount : 1
Sep 17, 2021 @ 18:13:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:13:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:13:27.452 [subscription-refresh][TEuPKbATIx] releaseShard 1 because exceed balance
Sep 17, 2021 @ 18:13:27.452 [subscription-refresh][TEuPKbATIx] releaseCount : 1
Sep 17, 2021 @ 18:12:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:12:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:11:45.988 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:11:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:11:35.550 [subscription-refresh][YEEjJO066v] acquireCount : 1
Sep 17, 2021 @ 18:11:35.549 [subscription-refresh][YEEjJO066v] acquireShard 3 because under balance
Sep 17, 2021 @ 18:11:26.971 [subscription-refresh][TEuPKbATIx] acquireCount : 1
Sep 17, 2021 @ 18:11:26.971 [subscription-refresh][TEuPKbATIx] acquireShard 1 because under balance
Sep 17, 2021 @ 18:10:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:10:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:09:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:09:43.124 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:08:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:08:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:08:35.516 [subscription-refresh][YEEjJO066v] releaseCount : 1
Sep 17, 2021 @ 18:08:35.516 [subscription-refresh][YEEjJO066v] releaseShard 1 because exceed balance
Sep 17, 2021 @ 18:07:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:07:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:07:35.277 [subscription-refresh][YEEjJO066v] acquireCount : 1
Sep 17, 2021 @ 18:07:35.276 [subscription-refresh][YEEjJO066v] acquireShard 1 because under balance
Sep 17, 2021 @ 18:06:46.115 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:06:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:05:45.978 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:05:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:04:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:04:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:03:46.001 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:03:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:02:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:02:43.177 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:01:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:01:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 18:00:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 18:00:43.177 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:59:46.130 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:59:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:58:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:58:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:57:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:57:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:56:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:56:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:55:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:55:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:54:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:54:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:53:45.948 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:53:43.080 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:52:46.010 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:52:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:51:45.935 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:51:42.966 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:50:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:50:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:50:24.601 [subscription-refresh][TEuPKbATIx] releaseCount : 1
Sep 17, 2021 @ 17:50:24.601 [subscription-refresh][TEuPKbATIx] releaseShard 1 because exceed balance
Sep 17, 2021 @ 17:49:46.107 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:49:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:49:18.176 [subscription-refresh][TEuPKbATIx] acquireShard 1 because under balance
Sep 17, 2021 @ 17:49:18.176 [subscription-refresh][TEuPKbATIx] acquireCount : 1
Sep 17, 2021 @ 17:48:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:48:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:47:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:47:42.966 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:46:46.044 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:46:42.966 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:45:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:45:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:44:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:44:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:43:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:43:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:42:46.144 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:42:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:41:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:41:42.966 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:40:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:40:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:39:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:39:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:38:46.148 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:38:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:37:46.092 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:37:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:36:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:36:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:35:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:35:42.963 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:34:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:34:43.029 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:33:45.931 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:33:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:32:45.937 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:32:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:31:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:31:42.965 [subscription-refresh][TEuPKbATIx] healthCheck
Sep 17, 2021 @ 17:30:45.932 [subscription-refresh][YEEjJO066v] healthCheck
Sep 17, 2021 @ 17:30:43.158 [subscription-refresh][TEuPKbATIx] healthCheck

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment