Lister.kt

package sandbox

import jetbrains.exodus.entitystore.PersistentEntityStoreImpl
import jetbrains.exodus.env.EnvironmentConfig
import jetbrains.exodus.env.EnvironmentImpl
import jetbrains.exodus.env.Environments
import jetbrains.exodus.io.FileDataReader
import jetbrains.exodus.log.LogConfig
import kotlinx.coroutines.experimental.future.await
import kotlinx.coroutines.experimental.runBlocking
import mu.KLogging
import software.amazon.awssdk.core.AwsRequestOverrideConfig
import software.amazon.awssdk.core.auth.AwsCredentials
import software.amazon.awssdk.core.auth.StaticCredentialsProvider
import software.amazon.awssdk.core.client.builder.ClientAsyncHttpConfiguration
import software.amazon.awssdk.core.regions.Region
import software.amazon.awssdk.http.nio.netty.NettySdkHttpClientFactory
import software.amazon.awssdk.services.s3.S3AdvancedConfiguration
import software.amazon.awssdk.services.s3.S3AsyncClient
import software.amazon.awssdk.services.s3.model.ListBucketsRequest
import software.amazon.awssdk.services.s3.model.ListObjectsRequest
import java.net.URI

fun main(args: Array<String>) {
    runBlocking {
        Lister.awsMain()
    }
}

object Lister : KLogging() {
    val bucketName: String = "youtrack"

    suspend fun awsMain() { // TODO: add AbortIncompleteMultipartUpload
        val region = Region.US_WEST_2
        val httpClient = NettySdkHttpClientFactory.builder().build().createHttpClient()

        val host = "172.27.121.165"
        val port = 9000

        val s3 = S3AsyncClient.builder()
                .asyncHttpConfiguration(
                        ClientAsyncHttpConfiguration.builder().httpClient(httpClient).build()
                )
                .region(region)
                .endpointOverride(URI("http://$host:$port"))
                .advancedConfiguration(S3AdvancedConfiguration.builder().pathStyleAccessEnabled(true).build()) // for tests only
                .credentialsProvider(StaticCredentialsProvider.create(AwsCredentials.create("XXX", "YYY")))
                .build()

        val extraHost = AwsRequestOverrideConfig.builder().header("Host", "$host:$port").build() // for tests only? maybe amazon ignores RFC

        doDebug(s3, extraHost)

        val pageSize = EnvironmentConfig.DEFAULT.logCachePageSize
        // val fileSize = EnvironmentConfig.DEFAULT.logFileSize
        val envKey = "teamsysstore"

        val reader = S3DataReader(s3, bucketName, extraHost)
        val config = EnvironmentConfig().setLogCachePageSize(pageSize).setGcEnabled(false).setEnvIsReadonly(true)

        val env = Environments.newInstance(LogConfig.create(reader, ReadOnlyDataWriter), config) as EnvironmentImpl

        PersistentEntityStoreImpl(env, envKey).use {
            it.executeInReadonlyTransaction {
                println("Issues: ${it.getAll("Issue").size()}")

                println("Drafts: ${it.findWithLinks("Issue", "draftOwner").size()}")
            }
        }

        logger.info { String.format("Read bytes: %8.2fKB", reader.bytesRead.toDouble() / 1024) } // 237877.13KB
    }

    private suspend fun doDebug(s3: S3AsyncClient, extraHost: AwsRequestOverrideConfig?) {
        logger.info { "Listing buckets\n" }
        val bucketsResponse = s3.listBuckets(ListBucketsRequest.builder().requestOverrideConfig(extraHost).build()).await()

        logger.info {
            buildString {
                append("Buckets list:\n")
                for (bucket in bucketsResponse.buckets()) {
                    append("    ")
                    append(bucket.name())
                    append('\n')
                }
            }
        }

        logger.info { "Listing objects\n" }

        val objectListing = s3.listObjects(ListObjectsRequest.builder()
                .requestOverrideConfig(extraHost)
                .bucket(bucketName)
                .delimiter("/")
                //.prefix("My")
                .build()).await()
        logger.info {
            buildString {
                append("Objects list:\n")
                for (objectSummary in objectListing.contents()) {
                    append("    ")
                    append(objectSummary.key())
                    append(" ")
                    append("(size = ")
                    append(objectSummary.size())
                    append(")\n")
                }
            }
        }
    }
}