ECS. AutoScaling config, Service placement, container limits in Kotlin

aws_sdk_v2_ecs_config.kt

import com.fasterxml.jackson.annotation.JsonPropertyOrder
import com.fasterxml.jackson.dataformat.csv.CsvMapper
import software.amazon.awssdk.services.ecs.EcsClient
import software.amazon.awssdk.services.ecs.model.DescribeClustersRequest
import software.amazon.awssdk.services.ecs.model.DescribeServicesRequest
import software.amazon.awssdk.services.ecs.model.DescribeTaskDefinitionRequest
import software.amazon.awssdk.services.ecs.model.ListServicesRequest
import java.util.concurrent.ForkJoinPool
import java.util.concurrent.TimeUnit
import kotlin.streams.asSequence
import kotlin.streams.toList

/* TODO Fix warnings below:
SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
SLF4J: Defaulting to no-operation (NOP) logger implementation
SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
*/

fun main() {
    val startTime = System.nanoTime()
    // More here: https://stackoverflow.com/a/21172732
    // The parallel streams use the default ForkJoinPool.commonPool
    // which by default has one less threads as you have processors,
    // as returned by Runtime.getRuntime().availableProcessors()

    // val cpuCount = Runtime.getRuntime().availableProcessors()
    // cpuCount: 12

    // val confParallelism = System.getProperty("java.util.concurrent.ForkJoinPool.common.parallelism")
    // confParallelism: null

    // val commonPoolParallelism = java.util.concurrent.ForkJoinPool.getCommonPoolParallelism()
    // commonPoolParallelism: 11

    val ecs = EcsClient.create()
    val containerConfigStream = ecs.describeClusters(
        DescribeClustersRequest.builder()
            .clusters(ecs.listClustersPaginator().clusterArns().toList())
            .build())
        .clusters().parallelStream().flatMap { ecsCluster ->
            ecs.listServicesPaginator(
                ListServicesRequest.builder()
                    .cluster(ecsCluster.clusterArn())
                    .build())
                .serviceArns().chunked(10).parallelStream().flatMap { ecsServiceArns ->
                    ecs.describeServices(
                        DescribeServicesRequest.builder()
                            .cluster(ecsCluster.clusterArn())
                            .services(ecsServiceArns)
                            .build())
                        .services().parallelStream().map { ecsService ->
                            val ecsTask = ecs.describeTaskDefinition(
                                DescribeTaskDefinitionRequest.builder()
                                    .taskDefinition(ecsService.taskDefinition())
                                    .build())
                                .taskDefinition()

                            val placementStrategy1 = ecsService.placementStrategy().getOrNull(0)
                            val placementStrategy2 = ecsService.placementStrategy().getOrNull(1)
                            val container = ecsTask.containerDefinitions()[0]
                            val ulimit1 = container.ulimits().getOrNull(0)
                            val ulimit2 = container.ulimits().getOrNull(1)
                            val ulimit3 = container.ulimits().getOrNull(2)

                            ContainerConfig(
                                awsAccountName = "TODO",
                                clusterName = ecsCluster.clusterName(),
                                serviceName = ecsService.serviceName(),
                                placementStrategy1Type = placementStrategy1?.typeAsString(),
                                placementStrategy1Field = placementStrategy1?.field(),
                                placementStrategy2Type = placementStrategy2?.typeAsString(),
                                placementStrategy2Field = placementStrategy2?.field(),
                                taskDefinitionFamily = ecsTask.family(),
                                containerDefinitionName = container.name(),
                                containerDefinitionMemoryMb = container.memory(),
                                containerDefinitionMemoryReservationMb = container.memoryReservation(),
                                containerDefinitionUlimit1Name = ulimit1?.nameAsString(),
                                containerDefinitionUlimit1SoftLimit = ulimit1?.softLimit(),
                                containerDefinitionUlimit1HardLimit = ulimit1?.hardLimit(),
                                containerDefinitionUlimit2Name = ulimit2?.nameAsString(),
                                containerDefinitionUlimit2SoftLimit = ulimit2?.softLimit(),
                                containerDefinitionUlimit2HardLimit = ulimit2?.hardLimit(),
                                containerDefinitionUlimit3Name = ulimit3?.nameAsString(),
                                containerDefinitionUlimit3SoftLimit = ulimit3?.softLimit(),
                                containerDefinitionUlimit3HardLimit = ulimit3?.hardLimit()
                            )
                        }.toList().stream()
                }.toList().stream()
        }


    val forkJoinPool = ForkJoinPool(12)
    val containerConfigs =
        forkJoinPool.submit(fun (): List<ContainerConfig> { return containerConfigStream.toList() }).get()

    // 6 secs: ForkJoinPool(12), parallelStream in each map/flatMap step, evaluate intermediate steps with .toList
    // 32 secs: ForkJoinPool(1), Stream<ContainerConfig>.toList()
    // 18 secs: ForkJoinPool(12), Stream<ContainerConfig>.toList()
    // 18 secs: ForkJoinPool(40), Stream<ContainerConfig>.toList()

    // 29 secs: Stream<ContainerConfig>.asSequence().asIterable(), then pass to CSV writer.
    // 17 secs: Stream<ContainerConfig>.toList(), then pass to CSV writer.

    val mapper = CsvMapper()
    val tsvSchema = mapper.schemaFor(ContainerConfig::class.java)
        .withHeader()
        .withColumnSeparator('\t')
        .withoutQuoteChar()
    mapper
        .writer(tsvSchema)
        .writeValues(System.out)
        .writeAll(containerConfigs)

    val timeTakenSec = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTime)
    println("DONE in $timeTakenSec secs")
}


@JsonPropertyOrder(
    "awsAccountName",
    "clusterName",
    "serviceName",
    "placementStrategy1Type",
    "placementStrategy1Field",
    "placementStrategy2Type",
    "placementStrategy2Field",
    "taskDefinitionFamily",
    "containerDefinitionName",
    "containerDefinitionMemoryMb",
    "containerDefinitionMemoryReservationMb",
    "containerDefinitionUlimit1Name",
    "containerDefinitionUlimit1SoftLimit",
    "containerDefinitionUlimit1HardLimit",
    "containerDefinitionUlimit2Name",
    "containerDefinitionUlimit2SoftLimit",
    "containerDefinitionUlimit2HardLimit",
    "containerDefinitionUlimit3Name",
    "containerDefinitionUlimit3SoftLimit",
    "containerDefinitionUlimit3HardLimit")
data class ContainerConfig(
    val awsAccountName: String,
    val clusterName: String,
    val serviceName: String,
    val placementStrategy1Type: String?,
    val placementStrategy1Field: String?,
    val placementStrategy2Type: String?,
    val placementStrategy2Field: String?,
    val taskDefinitionFamily: String,
    val containerDefinitionName: String,
    val containerDefinitionMemoryMb: Int?,
    val containerDefinitionMemoryReservationMb: Int?,
    val containerDefinitionUlimit1Name: String?,
    val containerDefinitionUlimit1SoftLimit: Int?,
    val containerDefinitionUlimit1HardLimit: Int?,
    val containerDefinitionUlimit2Name: String?,
    val containerDefinitionUlimit2SoftLimit: Int?,
    val containerDefinitionUlimit2HardLimit: Int?,
    val containerDefinitionUlimit3Name: String?,
    val containerDefinitionUlimit3SoftLimit: Int?,
    val containerDefinitionUlimit3HardLimit: Int?
)