Created
December 11, 2023 18:07
-
-
Save itudoben/40d1b845f02d48197d5c680e8ec43137 to your computer and use it in GitHub Desktop.
new get gradle propertie
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| /* | |
| * Copyright (c) 2017-2020, Sindice Limited. All Rights Reserved. | |
| * | |
| * This file is part of the Siren Federate Plugin project. | |
| * | |
| * The Siren Federate Plugin project is not open-source software. It is owned by Sindice Limited. The Siren Federate | |
| * Plugin project can not be copied and/or distributed without the express permission of Sindice Limited. Any form of | |
| * modification or reverse-engineering of the Siren Federate Plugin project is forbidden. | |
| */ | |
| package io.siren.federate.benchmark | |
| import com.google.api.gax.paging.Page | |
| import com.google.auth.Credentials | |
| import com.google.auth.oauth2.ServiceAccountCredentials | |
| import com.google.cloud.storage.Blob | |
| import com.google.cloud.storage.Storage | |
| import com.google.cloud.storage.StorageOptions | |
| import com.google.common.base.Preconditions | |
| import com.vdurmont.semver4j.Semver | |
| import groovy.json.JsonSlurper | |
| import groovy.text.GStringTemplateEngine | |
| import groovy.text.TemplateEngine | |
| import groovy.util.slurpersupport.GPathResult | |
| import io.siren.federate.FederateSystemProperty | |
| import io.siren.federate.benchmark.cluster.ClusterInfo | |
| import io.siren.federate.benchmark.cluster.ClusterPluginExtension | |
| import io.siren.federate.benchmark.cluster.ClusterTask | |
| import io.siren.federate.benchmark.cluster.ClusterType | |
| import io.siren.federate.benchmark.cluster.kubernetes.KubernetesConfig | |
| import io.siren.federate.benchmark.dataset.DatasetPlugin | |
| import io.siren.federate.benchmark.dataset.DatasetPluginExtension | |
| import io.siren.federate.benchmark.dataset.DatasetType | |
| import io.siren.federate.benchmark.dataset.SnapshotStorageType | |
| import io.siren.federate.benchmark.storage.StorageType | |
| import io.siren.federate.benchmark.test.jmeter.metricsplugin.JMeterTestCasePropertiesExtension | |
| import io.siren.federate.common.BenchmarkConstants | |
| import io.siren.federate.common.CommunicationUtils | |
| import io.siren.federate.common.FederateFullyQualifiedVersion | |
| import io.siren.federate.common.ProcessUtils | |
| import io.siren.federate.common.SnapshotName | |
| import io.siren.federate.common.Utils | |
| import io.siren.http.HttpClient | |
| import io.vavr.control.Either | |
| import org.apache.http.HttpHost | |
| import org.apache.http.HttpResponse | |
| import org.apache.http.HttpStatus | |
| import org.apache.http.client.methods.HttpGet | |
| import org.apache.http.client.methods.HttpPut | |
| import org.apache.http.entity.ContentType | |
| import org.apache.http.entity.StringEntity | |
| import org.apache.http.impl.client.HttpClients | |
| import org.apache.http.util.EntityUtils | |
| import org.gradle.api.GradleException | |
| import org.gradle.api.Project | |
| import org.slf4j.Logger | |
| import org.slf4j.LoggerFactory | |
| import javax.annotation.Nonnull | |
| import javax.annotation.Nullable | |
| import java.nio.file.Path | |
| import java.nio.file.Paths | |
| import java.time.ZoneId | |
| import java.time.ZonedDateTime | |
| import java.time.format.DateTimeFormatter | |
| import java.util.function.Consumer | |
| import java.util.function.Function | |
| import java.util.function.Supplier | |
| import java.util.logging.Level | |
| import java.util.stream.Stream | |
| import static io.siren.federate.benchmark.ExecutionEnvironment.CUSTOM | |
| import static io.siren.federate.benchmark.ExecutionEnvironment.DEVELOPMENT | |
| import static io.siren.federate.benchmark.ExecutionEnvironment.PRODUCTION | |
| import static io.siren.federate.benchmark.ExecutionEnvironment.RELEASE_CANDIDATE | |
| import static io.siren.federate.benchmark.ExecutionEnvironment.valueOf | |
| import static io.siren.federate.benchmark.ExecutionEnvironment.values | |
| import static io.siren.federate.common.BenchmarkConstants.ENV_ARTIFACTORY_API_KEY | |
| import static io.siren.federate.common.BenchmarkConstants.GCS_SERVICE_ACCOUNT_FILE_PROPERTY | |
| import static io.siren.federate.common.BenchmarkConstants.PROP_ARTIFACTORY_API_KEY | |
| import static io.siren.federate.common.BenchmarkConstants.PROP_BENCHMARK_RESULT_DIR | |
| import static io.siren.federate.common.BenchmarkConstants.PROP_BENCHMARK_UUID | |
| import static io.siren.federate.common.BenchmarkConstants.PROP_CIDR_BLOCK | |
| import static io.siren.federate.common.BenchmarkConstants.PROP_CLUSTER_ID | |
| import static io.siren.federate.common.BenchmarkConstants.PROP_CLUSTER_TYPE | |
| import static io.siren.federate.common.BenchmarkConstants.PROP_DRY_RUN | |
| import static io.siren.federate.common.BenchmarkConstants.PROP_ERP | |
| import static io.siren.federate.common.BenchmarkConstants.PROP_FEATURES | |
| import static io.siren.federate.common.BenchmarkConstants.PROP_FEDERATE_COMMIT | |
| import static io.siren.federate.common.BenchmarkConstants.PROP_TRACK_TOTAL_HITS | |
| import static io.siren.federate.common.BenchmarkConstants.PROP_USE_BUNDLED_JDK | |
| import static io.siren.federate.common.BenchmarkConstants.SIREN_MEMORY_ROOT_LIMIT_DATA | |
| import static io.siren.federate.common.BenchmarkConstants.SIREN_MEMORY_ROOT_LIMIT_MASTER | |
| final class BenchmarkContext { | |
| private static final Logger logger = LoggerFactory.getLogger(BenchmarkContext.class) | |
| public static final String PROP_SNAPSHOT_NAME_EXTENSION = 'snapshot.name.ext' | |
| private static final String PROP_DATASET_NAME = 'dataset.name' | |
| // This property is used to know if a spawned cluster can be reused for each scenario. | |
| private static final String PROP_GRADLE_CLUSTER_REUSE = 'cluster.spawned.reuse' | |
| private static final String PROP_CLUSTER_CUSTOM = 'cluster.nodes' | |
| private static final String PROP_CLUSTER_KUBERNETES = 'run.on.kubernetes' | |
| // This property allows to pass the branch to check out when installing gcloud-cicd Git Hub project. | |
| private static final String PROP_FEDERATE_FORCE_BUILD = 'federate.build.force' | |
| private static final String PROP_KUBERNETES_GIT_BRANCH = 'git.branch.kubernetes' | |
| private static final String NODES_SEPARATOR = ',' | |
| private static final String GRADLE_PROPERTY_RESULTS_PREFIX = 'results.property.' | |
| // States for the reused cluster | |
| private static final String STATE_NONE = 'none' | |
| private static final String STATE_START = 'start' | |
| private static final String STATE_STARTED = 'started' | |
| private static final String STATE_RUNNING = 'running' | |
| private static final String STATE_DELETING = 'deleting' | |
| private static final String STATE_DELETED = 'deleted' | |
| // The query parameter to enable tracking total hits | |
| private static final String URL_TRACK_TOTAL_HITS = '&track_total_hits=true' | |
| private static final int LIMITATION_INDEX_CHARACTERS = 63 | |
| private static final String KUBERNETES_ZONE = 'europe-west1-c' | |
| private static final String KUBERNETES_PROJECT = 'siren-cicd' | |
| private static final String DOWNLOADED_GIT_ARTIFACTS = 'federate-git-artifacts' | |
| private String bUuid | |
| int numberOfIndices | |
| int dataNodeCount | |
| int masterNodeCount | |
| int dataHeap | |
| int dataVolumeSize | |
| int dataSystemMemory | |
| int masterHeap | |
| int masterSystemMemory | |
| int dataFederateRootLimit | |
| int masterFederateRootLimit | |
| Integer clusterPid | |
| String currentClusterId | |
| String kubernetesClusterName | |
| private String state = STATE_NONE | |
| private boolean requestToDelete = false | |
| // This boolean allows to start the cluster only once. | |
| private boolean canReusableClusterBeStarted = true | |
| // When many scenarios are run, the same spawned cluster could be reused. | |
| private boolean doReuseCluster = false | |
| // This is the default mode of execution. | |
| ExecutionEnvironment executionEnvironment = DEVELOPMENT | |
| /** | |
| * This method loads all context information regarding the cluster and benchmark into a {@link BenchmarkContext} object added as | |
| * an extra property to the specified project. One can access it using {@link #getContext(org.gradle.api.Project)}. | |
| * <p/> | |
| * This method pulls all information from the cluster sending few requests, depending on network this may take few seconds per query. | |
| * It cannot be called more than once or it will fail. | |
| * <p/> | |
| * Information on collected data is available at <a href="https://sirensolutions.atlassian.net/browse/FEDE-4533">fields to collect to metrics issue</a>. | |
| * <p/> | |
| * This method throws an {@link IllegalStateException} if called more than once. | |
| * @param project | |
| * @throws IllegalStateException | |
| */ | |
| static void configureBenchmarkContext(Project project) { | |
| BenchmarkContext context = getContext(project) | |
| if (context == null) { | |
| // Configure a unique benchmark context. | |
| project.rootProject.ext.benchmarkContext = new BenchmarkContext() | |
| context = project.rootProject.ext.benchmarkContext | |
| } | |
| // Set the execution mode | |
| if (project.hasProperty('production')) { | |
| // Deprecated message | |
| // TODO remove -Pproduction https://sirensolutions.atlassian.net/browse/FEDE-6161 | |
| CommunicationUtils.deprecateMessage(project, project.logger, true, | |
| "Gradle property 'production' is deprecated.", | |
| "Use instead the Gradle property '-Pmode=production'.", | |
| '', | |
| "The property 'production' will be removed in the next release.", | |
| "https://sirensolutions.atlassian.net/browse/FEDE-6161" | |
| ) | |
| context.executionEnvironment = PRODUCTION | |
| } | |
| else { | |
| try { | |
| context.executionEnvironment = getGradlePropertyValue( | |
| project, | |
| 'mode', | |
| { -> DEVELOPMENT }, | |
| Arrays.asList(values()), | |
| { x -> x == 'rc' ? RELEASE_CANDIDATE : valueOf(x.toUpperCase()) } | |
| ) | |
| } | |
| catch (IllegalArgumentException iae) { | |
| CommunicationUtils.failWithMessage(project, true, | |
| "An error occurred while setting the mode.", | |
| '', | |
| iae.message, | |
| '', | |
| "Gradle property 'mode' must be set to one of '" + values().flatten().stream().map() { x -> x.name().toLowerCase() }.collect().join('\', \'') + "' modes." | |
| ) | |
| } | |
| } | |
| // Check whether it is reusing the cluster | |
| def doReuseCluster = project.rootProject.getProperties()[PROP_GRADLE_CLUSTER_REUSE] != null | |
| if (doReuseCluster) { | |
| context.setReuseCluster() | |
| } | |
| // Force Kubernetes cluster to be killed | |
| final String value = getGradlePropertyValue(project, 'kubernetes.cluster.force.kill', { -> | |
| null | |
| }, { -> }, { x -> x }) | |
| if (value != null) { | |
| forceKillKubernetes(project) | |
| } | |
| // Check gcloud-cicd settings | |
| String branch = project.getProperties()[PROP_KUBERNETES_GIT_BRANCH] | |
| if (branch == null) { | |
| branch = 'main' | |
| } | |
| final Path path = Paths.get(branch) | |
| final File gcloudCicdDir = path.toFile() | |
| if (gcloudCicdDir.exists() && gcloudCicdDir.isDirectory()) { | |
| // This directory cannot be removed as it was provided. | |
| context.isSirenGCloudCicdInstallDirProvided = true | |
| context.sirenGCloudCicdInstallDir = path | |
| } | |
| else { | |
| // On-demand get gcloud-cicd from Git but not until it is needed. | |
| context.isSirenGCloudCicdInstallDirProvided = false | |
| context.sirenGCloudCicdInstallDir = getDownloadsDir(project, 'gcloud-cicd') | |
| context.sirenGCloudCicdInstallDirBranch = branch | |
| } | |
| } | |
| /** | |
| * This function returns the benchmark context; null if it has not been configured. | |
| * @param project | |
| * @return the benchmark context; null if it has not been loaded using {@link #configureBenchmarkContext(org.gradle.api.Project)}. | |
| */ | |
| static @Nullable | |
| BenchmarkContext getContext(Project project) { | |
| if (project.rootProject.ext.has('benchmarkContext')) { | |
| project.rootProject.ext.benchmarkContext | |
| } | |
| else { | |
| null | |
| } | |
| } | |
| /** | |
| * @param project | |
| * @return the cluster ID from the Java system property if it exists; null otherwise. | |
| */ | |
| static @Nullable | |
| String getClusterIdSystemProperty(Project project) { | |
| FederateSystemProperty.createString(PROP_CLUSTER_ID, null, null).get() | |
| } | |
| static Path getPathInBenchmarkResults(Project project, String... aName) { | |
| // Support grouping results; see https://sirensolutions.atlassian.net/browse/FEDE-5604 | |
| getGradlePropertyValue( | |
| project, | |
| PROP_BENCHMARK_RESULT_DIR, | |
| { -> Paths.get(System.getenv('HOME'), "benchmark-${getBenchmarkUuid(project)}") }, | |
| { -> }, | |
| { x -> Paths.get(x, aName) } | |
| ) | |
| } | |
| /** | |
| * This method returns the {@link #URL_TRACK_TOTAL_HITS} in the optional by default or set to true. | |
| * If the Gradle property {@link BenchmarkConstants#PROP_TRACK_TOTAL_HITS} is set to false | |
| * then it returns a null {@link Optional}. | |
| */ | |
| static Optional<String> trackTotalHits(Project project) { | |
| if (project.hasProperty(PROP_TRACK_TOTAL_HITS)) { | |
| Optional.ofNullable(Boolean.parseBoolean(project.property(PROP_TRACK_TOTAL_HITS).toString()) ? URL_TRACK_TOTAL_HITS : null) | |
| } | |
| else { | |
| Optional.of(URL_TRACK_TOTAL_HITS) | |
| } | |
| } | |
| private static final String INDEX_NAME_DEVELOPMENT_PROPERTY = 'dev.metrics.index' | |
| private static final String INDEX_NAME_PRODUCTION = 'jmeter-federate' | |
| private static final String INDEX_NAME_DEV_PREFIX = "dev-federate-" | |
| /** | |
| * The index name has to be formatted as per Elasticsearch (ES) requirements: | |
| * https://www.elastic.co/guide/en/elasticsearch/guide/master/_document_metadata.html#_index | |
| * | |
| * @param name | |
| * @return | |
| */ | |
| private static String formatElasticsearchIndexName(final Optional<String> name) { | |
| // Note: the name must not contains commas, etc, | |
| // This could be enforced later as needed. | |
| if (name.isEmpty()) { | |
| return INDEX_NAME_DEV_PREFIX + System.getProperty('user.name') | |
| } | |
| final String nameLC = name.get().toLowerCase() | |
| if (nameLC.startsWith(INDEX_NAME_DEV_PREFIX)) { | |
| if (nameLC.length() == INDEX_NAME_DEV_PREFIX.length()) { | |
| return INDEX_NAME_DEV_PREFIX + System.getProperty('user.name') | |
| } | |
| return nameLC | |
| } | |
| return INDEX_NAME_DEV_PREFIX + nameLC | |
| } | |
| static String getFeatureFlagsString(Project project) { | |
| return getFeatureFlags(project).join(',') | |
| } | |
| /** | |
| * Returns the feature flags that were set using the {@link BenchmarkConstants#PROP_FEATURES} property. | |
| * The flags are trimmed, upper-cased, deduplicated, and sorted alphabetically. | |
| * If no flags are provided, this returns an empty array. | |
| */ | |
| static String[] getFeatureFlags(Project project) { | |
| getGradlePropertyValue( | |
| project, | |
| PROP_FEATURES, | |
| { -> '' }, | |
| { -> }, | |
| { features -> Arrays.stream(features.split(',')).map(String::trim).filter(i -> !i.isEmpty()).map(String::toUpperCase).sorted().distinct().toArray(String[]::new) } | |
| ) | |
| } | |
| static String getMetricsIndex(Project project) { | |
| final Optional<String> indexName = | |
| getGradlePropertyValue( | |
| project, | |
| INDEX_NAME_DEVELOPMENT_PROPERTY, | |
| { -> Optional.empty() }, | |
| { -> }, | |
| { x -> Optional.of(x) } | |
| ) | |
| String indexNameProposal | |
| //noinspection GroovyFallthrough | |
| switch (getContext(project).executionEnvironment) { | |
| case PRODUCTION: | |
| indexNameProposal = (indexName.isEmpty() || indexName.get().equalsIgnoreCase(INDEX_NAME_PRODUCTION)) ? | |
| INDEX_NAME_PRODUCTION : | |
| formatElasticsearchIndexName(indexName) | |
| break | |
| case RELEASE_CANDIDATE: | |
| case DEVELOPMENT: | |
| case CUSTOM: | |
| default: | |
| // Reference https://sirensolutions.atlassian.net/browse/DEV-526 | |
| // For any other purpose than production, anyone can use a `dev-federate-` prefixed index name. | |
| // A specified index name prefixed with `dev-federate-` as a Gradle `dev.metrics.index` property is also a valid non-production name. | |
| // When no dev index name is defined, then `dev-federate-<username>` index is returned. | |
| indexNameProposal = formatElasticsearchIndexName(indexName) | |
| } | |
| // Elasticsearch limits index name to 255 bytes. A single character could be multiple bytes. | |
| // To keep it simple and safe, limits index name length to LIMITATION_INDEX_CHARACTERS characters. | |
| if (indexNameProposal.length() > LIMITATION_INDEX_CHARACTERS) { | |
| indexNameProposal = indexNameProposal.substring(0, LIMITATION_INDEX_CHARACTERS) | |
| } | |
| return indexNameProposal | |
| } | |
| static boolean isIndexProduction(String metricsIndex) { | |
| metricsIndex == INDEX_NAME_PRODUCTION | |
| } | |
| static void assertNotIndexProduction(String metricsIndex) { | |
| assert !isIndexProduction(metricsIndex): "$metricsIndex cannot be named after 'metrics.siren.io' production index called '$INDEX_NAME_PRODUCTION'." | |
| } | |
| static String getMetricsSirenIoURL() { | |
| 'https://metrics.siren.io:9200' | |
| } | |
| static void assertMetricsUserCredentialsFromEnvExist() { | |
| assert getMetricsBenchmarkUsername() != null && getMetricsBenchmarkPassword() != null: | |
| "Shell environment variable 'METRICS_USERNAME' or 'METRICS_PASSWORD' must be set to access Federate at metrics.siren.io:9200" | |
| } | |
| static String getMetricsBenchmarkUsername() { | |
| 'federate_benchmarks' | |
| } | |
| static String getMetricsBenchmarkPassword() { | |
| return System.getenv(JMeterTestCasePropertiesExtension.ENV_METRICS_FEDERATE_PASSWORD) | |
| } | |
| static boolean forceBuildDataset(Project project) { | |
| final String value = getGradlePropertyValue(project, 'force.build.dataset', { -> | |
| null | |
| }, { -> }, { x -> x }) | |
| value != null | |
| } | |
| static File createKubernetesEnvironmentFile(Project project) { | |
| // Support grouping results and results dir; see https://sirensolutions.atlassian.net/browse/FEDE-5604 | |
| final File k8sEnvFile = getPathInBenchmarkResults(project.getRootProject(), "federate-${getFederateVersion(project, true)}.env").toFile() | |
| if (k8sEnvFile.exists()) { | |
| return k8sEnvFile | |
| } | |
| if (!k8sEnvFile.getParentFile().exists() && !k8sEnvFile.getParentFile().mkdirs()) { | |
| throw new RuntimeException('Cannot create ' + k8sEnvFile.getParentFile().toString()) | |
| } | |
| // Set the Kubernetes cluster name | |
| if (getContext(project).kubernetesClusterName == null) { | |
| throw new GradleException("BenchmarkContext.kubernetesClusterName must be set.") | |
| } | |
| final Map<String, String> binding = getKubernetesConfig(project) | |
| TemplateEngine engine = new GStringTemplateEngine() | |
| String text = engine.createTemplate(new InputStreamReader(BenchmarkContext.class.getClassLoader().getResourceAsStream('federate-kubernetes.template'))) | |
| .make(binding) | |
| .toString() | |
| k8sEnvFile.write(text, 'utf8') | |
| k8sEnvFile | |
| } | |
| static boolean doesClusterSupportRuntimeField(Project project) { | |
| // https://www.elastic.co/guide/en/elasticsearch/reference/7.11/runtime.html | |
| final Semver esClusterVersion = new Semver(getElasticsearchVersionFromFederateCommit(project), Semver.SemverType.LOOSE) | |
| esClusterVersion >= new Semver('7.11.0', Semver.SemverType.LOOSE) | |
| } | |
| static void setSnapshotRequestBody(Project project, HttpPut put) { | |
| final String esFedFromCommit = getElasticsearchVersionFromFederateCommit(project) | |
| final Semver esClusterVersion = new Semver(esFedFromCommit, Semver.SemverType.LOOSE) | |
| String settings | |
| // https://www.elastic.co/guide/en/elasticsearch/reference/7.12/get-features-api.html | |
| if (esClusterVersion >= new Semver('7.12.0', Semver.SemverType.LOOSE)) { | |
| settings = '{ "indices": ["join*", "child*", "parent*"], "ignore_unavailable": true, "include_global_state": false, "feature_states": ["none"]}' | |
| } | |
| else { | |
| settings = '{ "indices": ["join*", "child*", "parent*"], "ignore_unavailable": true, "include_global_state": false}' | |
| } | |
| put.setEntity(new StringEntity(settings, ContentType.APPLICATION_JSON)) | |
| } | |
| /** | |
| * This returns one of the {@link SnapshotStorageType}s. | |
| * <p> | |
| * To resolve the snapshot storage type, this method evaluates sequentially the rules below until one matches: | |
| * <p> | |
| * 1/ If the property {@link BenchmarkConstants#PROP_SNAPSHOT_STORAGE} is set, then its corresponding {@link SnapshotStorageType} is returned. | |
| * <p> | |
| * 2/ if the {@link DatasetPluginExtension#gcsSnapshotObjectName} is defined, then {@link SnapshotStorageType#GCLOUD} is returned. | |
| * <p> | |
| * 3/ Depending of the cluster type, the corresponding {@link SnapshotStorageType} is returned. | |
| * <p> | |
| * 4/ Last resort, {@link SnapshotStorageType#FILE_SYSTEM} is returned. | |
| */ | |
| static SnapshotStorageType getSnapshotStorageType(Project project) { | |
| // 1/ | |
| if (project.hasProperty(BenchmarkConstants.PROP_SNAPSHOT_STORAGE)) { | |
| return SnapshotStorageType.get(String.valueOf(project.property(BenchmarkConstants.PROP_SNAPSHOT_STORAGE)).toLowerCase()) | |
| } | |
| // 2/ | |
| def isGcsDefined = (project.extensions.findByName(DatasetPlugin.EXTENSION_NAME) as DatasetPluginExtension).isGcsSnapshotObjectNameDefined() | |
| if (isGcsDefined) { | |
| return SnapshotStorageType.GCLOUD | |
| } | |
| // 3/ | |
| ClusterType clusterType = getClusterType(project) | |
| switch (clusterType) { | |
| case ClusterType.GCLOUD: | |
| return SnapshotStorageType.GCLOUD | |
| case ClusterType.MOCK: | |
| return SnapshotStorageType.MOCK | |
| } | |
| // 4/ | |
| return SnapshotStorageType.FILE_SYSTEM | |
| } | |
| static String getTestScenarioId(Project project) { | |
| // Support grouping results by scenario; see https://sirensolutions.atlassian.net/browse/FEDE-5604 | |
| // Use a custom made prefix as the group name for all test plans in the same test plan specs. | |
| String tmpVar = project.projectDir.getName() | |
| tmpVar.split("[-_]").stream() | |
| .map { x -> x.charAt(0).toString() } | |
| .reduce("", String::concat) + | |
| tmpVar.replaceAll('[^0-9]', '') | |
| } | |
| // Set default sample size to 250 for all scenarios but inner join ones. | |
| // https://sirensolutions.atlassian.net/browse/FEDE-6736 | |
| private static final int SAMPLE_SIZE_DEFAULT = 250 | |
| private static final int SAMPLE_SIZE_INNER_JOIN = 25 | |
| static int getSampleSize(Project project) { | |
| // Keep sample size for inner join to 25 | |
| // https://sirensolutions.atlassian.net/browse/FEDE-6736 | |
| if (project.name.contains('inner-join')) { | |
| return SAMPLE_SIZE_INNER_JOIN | |
| } | |
| // Set the sample size for all JMeter test plan | |
| // https://sirensolutions.atlassian.net/browse/FEDE-6707 | |
| return getGradlePropertyValue(project, 'sample.size', { -> SAMPLE_SIZE_DEFAULT }, { -> }, | |
| { x -> Integer.valueOf(x).intValue() }) | |
| } | |
| static String getBenchmarkLabel(Project project) { | |
| // Implementation of https://sirensolutions.atlassian.net/browse/FEDE-6230 by recycling the metrics document field | |
| // called environment. | |
| // This gives a way to label a benchmark runs and retrieve them all in metrics.siren.io | |
| getGradlePropertyValue(project, 'benchmark.label', { -> 'benchmark.label' }, { -> }, { x -> x }) | |
| } | |
| static String getTimeNow() { | |
| ZonedDateTime.now(ZoneId.of('UTC')).format(DateTimeFormatter.ofPattern("EEE LLL dd HH:mm:ss 'UTC' yyyy")) | |
| } | |
| /** | |
| * @return true by default; or false from the Gradle property called 'benchmark.alias.creation'. | |
| */ | |
| static boolean createAlias(Project project) { | |
| getGradlePropertyValue(project, 'benchmark.alias.creation', { -> true | |
| }, [false, true], { x -> Boolean.valueOf(x) }) | |
| } | |
| void setStartReusableCluster() { | |
| state = STATE_START | |
| // Set this to false and never change it even if it's deleted. | |
| canReusableClusterBeStarted = false | |
| } | |
| boolean canStartReusableCluster() { | |
| state == STATE_START | |
| } | |
| void setReusableClusterStarted() { | |
| state = STATE_STARTED | |
| } | |
| void setReusableClusterRunning() { | |
| state = STATE_RUNNING | |
| } | |
| boolean isReusableClusterRunning() { | |
| state == STATE_RUNNING | |
| } | |
| void setReusableClusterDelete() { | |
| requestToDelete = true | |
| } | |
| void setReuseCluster() { | |
| doReuseCluster = true | |
| } | |
| boolean doReuseCluster() { | |
| doReuseCluster | |
| } | |
| boolean canDeleteCluster() { | |
| requestToDelete | |
| } | |
| void setClusterDeleting() { | |
| state = STATE_DELETING | |
| } | |
| boolean isClusterDeleting() { | |
| state == STATE_DELETING | |
| } | |
| void setClusterDeleted() { | |
| state = STATE_DELETED | |
| } | |
| /** | |
| * @return true if the reusable cluster can be started; otherwise false. | |
| */ | |
| boolean canReusableClusterBeStarted() { | |
| canReusableClusterBeStarted | |
| } | |
| /** | |
| * This method depends on Gradle properties and therefore fails during the Initialization and Configuration phases of a build. | |
| * See https://docs.gradle.org/current/userguide/build_lifecycle.html. | |
| * | |
| * This method must be called during the Execution phase of a build, so must be all methods calling this method. | |
| * Execution phase of a task happens in a task action (https://docs.gradle.org/current/userguide/build_lifecycle.html#sec:task_execution). | |
| * | |
| * @param project | |
| * @return a label with all lower case alphanumeric. | |
| */ | |
| static String getBenchmarkUuid(Project project) { | |
| if (getContext(project).bUuid == null) { | |
| String benchmarkUuid = getGradlePropertyValue( | |
| project, | |
| PROP_BENCHMARK_UUID, | |
| { -> | |
| String commit = getFederateCommit(project, false) | |
| commit == null ? null : (ZonedDateTime.now(ZoneId.of('UTC')).format(DateTimeFormatter.ofPattern('yyyyMMdd_HHmmss')) + '_' + commit) | |
| }, | |
| { -> }, | |
| { x -> x } | |
| ) | |
| if (benchmarkUuid != null) { | |
| // Keep it lowercase. | |
| getContext(project).bUuid = benchmarkUuid.toLowerCase() | |
| CommunicationUtils.infoMessage(project, logger, true, | |
| "Set benchmark context with benchmark UUID " + getContext(project).bUuid, | |
| ) | |
| } | |
| } | |
| if (getContext(project).bUuid == null) { | |
| String msg = "Cannot set the benchmark UUID; provide from the Gradle property '$PROP_BENCHMARK_UUID'." | |
| throw new GradleException(msg) | |
| } | |
| getContext(project).bUuid | |
| } | |
| private void setClusterMetadata(int numberOfIndices, int dataNodeCount, int masterNodeCount, int dataHeap, int masterHeap, int dataVolumeSize, | |
| int dataSystemMemory, int masterSystemMemory, int dataFederateRootLimit, int masterFederateRootLimit) { | |
| this.numberOfIndices = numberOfIndices | |
| this.dataNodeCount = dataNodeCount | |
| this.masterNodeCount = masterNodeCount | |
| this.dataHeap = dataHeap | |
| this.masterHeap = masterHeap | |
| this.dataVolumeSize = dataVolumeSize | |
| this.dataSystemMemory = dataSystemMemory | |
| this.masterSystemMemory = masterSystemMemory | |
| this.dataFederateRootLimit = dataFederateRootLimit | |
| this.masterFederateRootLimit = masterFederateRootLimit | |
| } | |
| static void setClusterMetadataFromTask(ClusterTask clusterTask, Project project) { | |
| if (isDryRun(project)) { | |
| return | |
| } | |
| // Get one IP and port to communicate with the cluster | |
| ClusterInfo clusterInfo = clusterTask.getClusterInfo().get() | |
| if (clusterInfo == null) { | |
| String msg = "One of the spawn cluster task must be called prior to access its cluster info." | |
| throw new IllegalStateException(msg) | |
| } | |
| clusterInfo.withFirstNode { hostname, port -> | |
| // Get number of indices | |
| int numberOfIndices = "http://$hostname:$port/_cat/indices".toURL().text.split('\n').size() | |
| String jsonString = "http://$hostname:$port/_cluster/stats?human".toURL().text | |
| def nodesStats = new JsonSlurper().parseText(jsonString).nodes | |
| int dataNodeCount = nodesStats.count.data | |
| int masterNodeCount = nodesStats.count.master | |
| int dataHeap = (Double.valueOf(nodesStats.jvm.mem.heap_max_in_bytes) / Math.pow(2D, 30D)).intValue() | |
| int masterHeap = (Double.valueOf(nodesStats.jvm.mem.heap_max_in_bytes) / Math.pow(2D, 30D)).intValue() | |
| int dataVolumeSize = nodesStats.fs.total_in_bytes ? (Double.valueOf(nodesStats.fs.total_in_bytes) / Math.pow(2D, 30D)).intValue() : 0 | |
| int dataSystemMemory = (Double.valueOf(nodesStats.os.mem.total_in_bytes) / Math.pow(2D, 30D)).intValue() | |
| int masterSystemMemory = (Double.valueOf(nodesStats.os.mem.total_in_bytes) / Math.pow(2D, 30D)).intValue() | |
| jsonString = "http://$hostname:$port/_siren/nodes/stats?pretty".toURL().text | |
| def sirenStats = new JsonSlurper().parseText(jsonString) | |
| // find first data node to get his allocated_root_memory_in_bytes | |
| // See https://docs.siren.io/siren-federate-user-guide/23/siren-federate/cluster-apis.html#siren-federate-cluster-apis-memory | |
| int dataFederateRootLimit = 0 | |
| int masterFederateRootLimit = 0 | |
| def ite = sirenStats.nodes.iterator() | |
| while (ite.hasNext()) { | |
| def node = ite.next().value | |
| if (node.roles.contains('data')) { | |
| dataFederateRootLimit = Integer.valueOf(node.memory.allocated_root_memory_in_bytes) | |
| break | |
| } | |
| if (node.roles.contains('master')) { | |
| masterFederateRootLimit = Integer.valueOf(node.memory.allocated_root_memory_in_bytes) | |
| break | |
| } | |
| } | |
| // Set the benchmark context | |
| getContext(project).setClusterMetadata(numberOfIndices, dataNodeCount, masterNodeCount, dataHeap, masterHeap, dataVolumeSize, | |
| dataSystemMemory, masterSystemMemory, dataFederateRootLimit, masterFederateRootLimit) | |
| } | |
| } | |
| static String getElasticsearchRepoPath(Project project) { | |
| project.findProperty(PROP_ERP) ?: '/tmp/snapshot-repository' | |
| } | |
| static String generateFederateBundleZipFileName(String elasticsearchVersion, String federateCommit) { | |
| "es-federate-${elasticsearchVersion}-${federateCommit}.zip".toString() | |
| } | |
| static String getArtifactoryApiKey(Project project) { | |
| // Check if the Gradle property is set. | |
| String key = project.hasProperty(PROP_ARTIFACTORY_API_KEY) ? project.property(PROP_ARTIFACTORY_API_KEY) : null | |
| if (key == null) { | |
| // Check user's environment. | |
| key = System.getenv().get(ENV_ARTIFACTORY_API_KEY) | |
| } | |
| if (key == null) { | |
| throw new GradleException("The Artifactory API key is not set.\nEither 1/ it is set through the Gradle property called ${PROP_ARTIFACTORY_API_KEY}, " + | |
| "or \n2/ through the user's environment constant called ${ENV_ARTIFACTORY_API_KEY}.") | |
| } | |
| logger.info "Found Artifactory API Key ending with '${key.substring(key.length() - 4, key.length())}'." | |
| key | |
| } | |
| private static Optional<String> requestFederateCommitFromCluster(hostname, port) { | |
| String[] jsonString = "http://$hostname:$port/_cat/plugins".toURL().text.split('\n') | |
| Optional<String> entryOpt = Stream.of(jsonString) | |
| .filter({ s -> s.contains('siren-federate') | |
| }) | |
| .map({ s -> s.split("\\s+")[2] | |
| }) | |
| .findFirst() | |
| if (entryOpt.isPresent()) { | |
| String[] all = entryOpt.get().split('-') | |
| // Expect format as es-data-1 siren-federate 7.17.4-27.3-c7a3cb378b96ee9156b786cb8106f925570b767a | |
| // or as es-data-1 siren-federate 7.17.4-27.4-SNAPSHOT-c7a3cb378b96ee9156b786cb8106f925570b767a | |
| // Sometimes it is as es-data-1 siren-federate 7.17.4-27.3 and cannot get the commit used. | |
| if (all.length >= 3) { | |
| Optional.of(all[all.length - 1]) | |
| } | |
| else { | |
| Optional.empty() | |
| } | |
| } | |
| else { | |
| Optional.empty() | |
| } | |
| } | |
| /** | |
| * Gets the federate commit identifier from project property {@link io.siren.federate.common.BenchmarkConstants#PROP_FEDERATE_COMMIT}. | |
| * <p> | |
| * If the property doesn't exist, an exception is thrown due to <a href="https://github.com/sirensolutions/siren-platform/issues/2801">issue 2801</a>. | |
| * @param fail if true throw a {@link GradleException}; otherwise the method warns with a message and returns null. | |
| */ | |
| static @Nullable | |
| String getFederateCommit(Project project, boolean fail = true) { | |
| if (isUsingAnExistingCluster(project)) { | |
| // As defined in benchmark / README.md / ## Executing benchmark on an existing cluster, the nodes information is passed | |
| // as <node IP>:<HTTP port>:<Transport port>. | |
| String[] nodeIpPorts = getNodes(project).iterator().next().split(':') | |
| Optional<String> res = requestFederateCommitFromCluster(nodeIpPorts[0], Integer.valueOf(nodeIpPorts[1]).intValue()) | |
| if (res.isPresent()) { | |
| return res.get() | |
| } | |
| } | |
| // There is no cluster running to talk to. | |
| String value = getFederateCommitFromGradleProperty(project) | |
| if (value == null || value.isEmpty()) { | |
| // TODO: issue https://github.com/sirensolutions/siren-platform/issues/2801 | |
| String msg = "Failed to get the federate Git SHA-1 commit.\nIt must be defined by passing to gradle command line a -P${PROP_FEDERATE_COMMIT}=<a git SHA-1 commit>." | |
| logger.warn(msg) | |
| if (fail) { | |
| throw new GradleException(msg) | |
| } | |
| null | |
| } | |
| else { | |
| value | |
| } | |
| } | |
| private static @Nullable | |
| String getFederateCommitFromGradleProperty(Project project) { | |
| getGradlePropertyValue(project, PROP_FEDERATE_COMMIT, { -> null }, { -> }, { x -> x }) | |
| } | |
| /** | |
| * The method checks the Gradle property (the one passed on command line with -PpropertyName=some-value). | |
| * <p/>The method returns some-value or the propertyDefaultValue, referred henceforth as the VALUE, which is processed as follows: | |
| * <p/>1/ valuesSupported = {@link #{ -> }}: returns the VALUE. | |
| * <p/>2/ valuesSupported != {@link #{ -> }}: the VALUE is matched against the list of valuesSupported as follows: | |
| * <p/> 2.1/ it returns the VALUE if it is supported. | |
| * <p/> 2.2/ it throws an {@link IllegalArgumentException} if the VALUE is not supported. | |
| * @param project | |
| * @param propertyName | |
| * @param propertyDefaultValue | |
| * @param valuesSupported | |
| * @param conversionFunction | |
| */ | |
| static <T_VALUE> T_VALUE getGradlePropertyValue( | |
| Project project, String propertyName, | |
| Supplier<T_VALUE> propertyDefaultValue, | |
| @Nullable List<T_VALUE> valuesSupported, | |
| Function<String, T_VALUE> conversionFunction | |
| ) { | |
| final Consumer<T_VALUE> validator = new Consumer<T_VALUE>() { | |
| @Override | |
| void accept(T_VALUE value) { | |
| if (valuesSupported != null && !valuesSupported.contains(value)) { | |
| String valuesTxt = valuesSupported.collect { T_VALUE it -> | |
| String.format("'%s': %s", it as String, it.class.getName()) | |
| }.join(", ") | |
| String msg = "The specified Gradle Property '${propertyName}' is defined as the value '${value}' with type ${value.class}" + | |
| " which is not supported." + | |
| " Please provide one from the available value to type: { ${valuesTxt} }." | |
| throw new IllegalArgumentException(msg) | |
| } | |
| } | |
| } | |
| getGradlePropertyValue(project, propertyName, propertyDefaultValue, validator, conversionFunction) | |
| } | |
| static <T_VALUE> T_VALUE getGradlePropertyValue( | |
| Project project, String propertyName, | |
| Supplier<T_VALUE> propertyDefaultValue, | |
| Consumer<T_VALUE> validator, | |
| Function<String, T_VALUE> conversionFunction | |
| ) { | |
| final T_VALUE value | |
| if (project.hasProperty(propertyName)) { | |
| value = conversionFunction.apply(project.property(propertyName) as String) | |
| } | |
| else { | |
| value = propertyDefaultValue.get() | |
| } | |
| validator.accept(value) | |
| value | |
| } | |
| /** | |
| * Gets cluster type from project property {@link io.siren.federate.common.BenchmarkConstants#PROP_CLUSTER_TYPE}. If the property doesn't exist the extension's value is | |
| * used. | |
| */ | |
| static ClusterType getClusterType(Project project) { | |
| final ClusterType clusterType | |
| if (isSpawningAKubernetesCluster(project)) { | |
| clusterType = ClusterType.KUBERNETES | |
| } | |
| else { | |
| clusterType = isUsingAnExistingCluster(project) ? ClusterType.CUSTOM : | |
| project.hasProperty(PROP_CLUSTER_TYPE) ? | |
| ClusterType.getEnum(String.valueOf(project.property(PROP_CLUSTER_TYPE))) as ClusterType : | |
| (project.extensions.findByName(ClusterPluginExtension.EXTENSION_NAME) as ClusterPluginExtension).clusterType | |
| } | |
| clusterType | |
| } | |
| /** | |
| * Checks the value of project property {@link io.siren.federate.common.BenchmarkConstants#PROP_USE_BUNDLED_JDK}. | |
| * <p/> | |
| * <a href="https://discuss.elastic.co/t/bundled-jdk-with-version-7-0-x/180684">Starting with 7.0</a>, Elasticsearch | |
| * comes with bundled JDK, therefore depending on the version of Elasticsearch used by the cluster, if the version is | |
| * lower than 7.0.0, this method returns false. | |
| * | |
| * @return true by default, or false if {@link io.siren.federate.common.BenchmarkConstants#PROP_USE_BUNDLED_JDK} is explicitly set to false. | |
| */ | |
| static Boolean doesClusterUseBundledJDK(Project project) { | |
| if (isFederateCommitUsingElasticsearchVersionLessThanOr_6(project)) { | |
| logger.warn("Elasticsearch version '{}' of the specified Federate commit '{}' does not support bundled JDK.\n" + | |
| "Elasticsearch version must be 7.0.0 or greater to use Gradle property '{}'.", | |
| getElasticsearchVersionFromFederateCommit(project), getFederateCommit(project), PROP_USE_BUNDLED_JDK) | |
| return false | |
| } | |
| if (project.hasProperty(PROP_USE_BUNDLED_JDK)) { | |
| return Boolean.valueOf(project.property(PROP_USE_BUNDLED_JDK).toString()) | |
| } | |
| return true | |
| } | |
| static String getProjectFullyQualifiedName(Project project) { | |
| (':' + project.rootProject.getName() + project.getPath()).replaceAll(':', '_') | |
| } | |
| /** | |
| * Please use instead {@link #isUsingAnExistingCluster(org.gradle.api.Project)}. | |
| */ | |
| @Deprecated | |
| static boolean useExistingCluster(Project project) { | |
| isUsingAnExistingCluster(project) | |
| } | |
| static boolean isDryRun(Project project) { | |
| project.getProperties()[PROP_DRY_RUN] != null | |
| } | |
| static boolean isUsingAnExistingCluster(Project project) { | |
| def t = !(System.getProperty(PROP_CLUSTER_CUSTOM) == null && project.getProperties()[PROP_CLUSTER_CUSTOM] == null) | |
| if (t) { | |
| printInfoOnExistingCluster(project, getNodesAsString(project)) | |
| } | |
| t | |
| } | |
| static boolean forceBuildFederate(Project project) { | |
| project.getProperties()[PROP_FEDERATE_FORCE_BUILD] != null | |
| } | |
| static boolean isSpawningAKubernetesCluster(Project project) { | |
| def t = !(System.getProperty(PROP_CLUSTER_KUBERNETES) == null && project.getProperties()[PROP_CLUSTER_KUBERNETES] == null) | |
| if (t) { | |
| printInfoOnKubernetesCluster(project) | |
| } | |
| t | |
| } | |
| static Iterable<String> getNodes(Project project) { | |
| String nodes = getNodesAsString(project) | |
| printInfoOnExistingCluster(project, nodes) | |
| Arrays.asList(nodes.split(NODES_SEPARATOR)) | |
| } | |
| private static String getNodesAsString(Project project) { | |
| // Grab the nodes | |
| String nodes = null | |
| if (System.getProperty(PROP_CLUSTER_CUSTOM) != null) { | |
| nodes = System.getProperty(PROP_CLUSTER_CUSTOM) | |
| } | |
| else { | |
| final def value = project.getProperties()[PROP_CLUSTER_CUSTOM] | |
| if (value != null) { | |
| nodes = value.toString() | |
| } | |
| } | |
| if (nodes == null) { | |
| throw new IllegalStateException("The property $PROP_CLUSTER_CUSTOM is not defined neither by Java system property nor by Gradle property." + | |
| "\nTherefore, we cannot get the nodes of the existing cluster from this property") | |
| } | |
| return nodes | |
| } | |
| static List<StorageType> getStorageTypes(Project project) { | |
| // Check whether to store results to metrics.siren.io or not. | |
| final Optional<String> propResultsStorage = Optional.ofNullable(project.getProperties()[BenchmarkConstants.PROP_RESULTS_STORAGE] as String ?: '') | |
| StorageType.getStorageTypesOrDefaults(propResultsStorage, ',') | |
| } | |
| private static void printInfoOnExistingCluster(Project project, String nodes) { | |
| CommunicationUtils.infoMessage(project, logger, true, | |
| "Java system property or Gradle property called $PROP_CLUSTER_CUSTOM is set to $nodes, ", | |
| "these will be used to communicate with the existing cluster.", | |
| "", | |
| "Note that the existing cluster won't be managed, that means:", | |
| " - no Federate bundle will be built nor published,", | |
| " - no cluster will be spawned,", | |
| " - no data will be restored from snapshot,", | |
| " - no dataset will be built and,", | |
| " - the cluster won't be destroyed.", | |
| "", | |
| "The cluster caches will be cleaned up after each scenario execution.") | |
| } | |
| private static void printInfoOnKubernetesCluster(Project project) { | |
| CommunicationUtils.infoMessage(project, logger, true, | |
| "Java system property or Gradle property called $PROP_CLUSTER_KUBERNETES is set, ", | |
| "this will spawn a cluster on kubernetes and run any benchmark local, gcloud or custom on it." | |
| ) | |
| } | |
| private static void errorMultipleClusterTypeAndFail(Project project) { | |
| CommunicationUtils.failWithMessage(project, true, "An issue occurred as both properties `$PROP_CLUSTER_CUSTOM` and `$PROP_CLUSTER_KUBERNETES` were provided regarding the cluster to use.", | |
| "The process cannot choose which cluster to use.", | |
| "", | |
| "Only one option is allowed to either run the benchmark on an existing cluster (i.e. $PROP_CLUSTER_CUSTOM) or,", | |
| "a kubernetes cluster is spawned prior to running a benchmark (i.e. $PROP_CLUSTER_KUBERNETES).", | |
| "", | |
| "Please pick one!" | |
| ) | |
| } | |
| /** | |
| * Get the download path for the specified paths. | |
| */ | |
| static Path getDownloadsDir(Project project, String... paths) { | |
| List<String> pathFull = [BenchmarkConstants.ELASTICSEARCH_BUNDLE_PLACEHOLDER, 'build', 'downloads'] | |
| pathFull.addAll(paths) | |
| Paths.get(project.rootDir.absolutePath, (String[]) pathFull.stream().toArray()) | |
| } | |
| /** | |
| * Get the workspace directory where the federate repository will be cloned into. This directory may not exist | |
| * yet, it's up to the users to check for its existence. | |
| * | |
| * @param project | |
| * @return the workspace directory | |
| */ | |
| static File getWorkspaceDir(Project project) { | |
| getDownloadsDir(project, 'siren-platform').toFile() | |
| } | |
| static Path getKubernetesClusterPath(Project project, String... paths) { | |
| List<String> pathFull = ['cluster', "kubernetes-${getBenchmarkUuid(project)}".toString()] | |
| pathFull.addAll(paths) | |
| Paths.get(project.rootProject.buildDir.toString(), (String[]) pathFull.stream().toArray()) | |
| } | |
| /** | |
| * @param project | |
| * @return the content of the POM for that the Federate commit specified as | |
| * {@link BenchmarkConstants#PROP_FEDERATE_COMMIT} on the command line. | |
| */ | |
| private static @Nullable | |
| GPathResult getPomOrDownloadItIfDoesNotExistUsingFederateCommit(Project project) { | |
| final String filePrefix = 'pom' | |
| final String fileExtension = 'xml' | |
| final String uriPath = "parent/${filePrefix}.${fileExtension}" | |
| final File targetFile = getOrDownloadItIfDoesNotExistUsingFederateCommit(project, filePrefix, fileExtension, uriPath) | |
| new XmlSlurper().parse(targetFile) | |
| } | |
| private static @Nullable | |
| File getOrDownloadItIfDoesNotExistUsingFederateCommit( | |
| Project project, String filePrefix, String fileExtension, String uriPath) { | |
| File workDir = getDownloadsDir(project, DOWNLOADED_GIT_ARTIFACTS).toFile() | |
| if (!workDir.exists() && !workDir.mkdirs()) { | |
| throw new GradleException("Failed to create the work directory ${workDir}.") | |
| } | |
| String federateCommit = getFederateCommit(project) | |
| if (federateCommit == null) { | |
| throw new GradleException("Unable to get the federate commit from the project.") | |
| } | |
| final File theFile = new File(workDir, "${filePrefix}_${federateCommit}.${fileExtension}") | |
| def needToDownloadTheTargetFile = !(theFile.exists()) | |
| if (needToDownloadTheTargetFile) { | |
| String hostname = 'api.github.com' | |
| int port = 443 | |
| String scheme = 'https' | |
| HttpHost target = new HttpHost(hostname, port, scheme) | |
| String gitUrl = "$target/repos/sirensolutions/siren-platform/contents/${uriPath}?ref=$federateCommit".toString() | |
| HttpGet request = new HttpGet(gitUrl) | |
| // This method downloads the URI path from GitHub for a particular Git commit if it does not exist. | |
| // The GitHub API is used with the specified GitHub token. | |
| Either<String, String> eitherRequest = HttpClient.executeHttpRequest(target, request, true) | |
| if (eitherRequest.isLeft()) { | |
| logger.error(gitUrl) | |
| throw new GradleException(eitherRequest.getLeft()) | |
| } | |
| String jsonString = eitherRequest.get() | |
| def json = new JsonSlurper().parseText(jsonString) | |
| if (json.status == HttpStatus.SC_NOT_FOUND || json.content == null) { | |
| throw new GradleException("Failed to download content from $gitUrl\nOutput: ${jsonString}") | |
| } | |
| String content = new String(Base64.getMimeDecoder().decode(json.content as String)) | |
| theFile.write(content) | |
| } | |
| theFile | |
| } | |
| static String getElasticsearchDistributionFormat() { | |
| 'tar.gz' | |
| } | |
| private static String getElasticsearchArtifactId(Project project) { | |
| 'elasticsearch' + (isFederateCommitUsingElasticsearchVersionLessThanOr_6(project) ? '-oss' : '') | |
| } | |
| private static String getElasticsearchDistributionVersion(Project project) { | |
| getElasticsearchVersionFromFederateCommit(project) + (isFederateCommitUsingElasticsearchVersionLessThanOr_6(project) ? '' : "-${Utils.getElasticsearchDistributionVersion()}".toString()) | |
| } | |
| static String getElasticsearchDownloadedDependencyFileName(Project project) { | |
| "${getElasticsearchArtifactId(project)}-${getElasticsearchDistributionVersion(project)}.${getElasticsearchDistributionFormat()}".toString() | |
| } | |
| static String getElasticsearchDependency(Project project) { | |
| "elastic:${getElasticsearchArtifactId(project)}:${getElasticsearchDistributionVersion(project)}@${getElasticsearchDistributionFormat()}".toString() | |
| } | |
| static boolean isFederateCommitUsingElasticsearchVersionLessThanOr_6(Project project) { | |
| String esVersion = getElasticsearchVersionFromFederateCommit(project) | |
| if (esVersion == null) { | |
| logger.warn("Ignoring esVersion from `getElasticsearchVersionFromFederateCommit` in `isFederateCommitUsingElasticsearchVersionLessThanOr_6` mostly at build-time.") | |
| return false | |
| } | |
| new Semver(esVersion).isLowerThan('7.0.0') | |
| } | |
| /** | |
| * @param project to get the Federate commit property from. | |
| * @param federateVersion 21.1, 19.3, 10.3.4, 24.0, etc. | |
| * @return -1 if Federate commit version is lower than specified version, 0 if the same and 1 if greater. | |
| */ | |
| static int compareFederateCommitVersionToFederateVersion(Project project, String federateVersion) { | |
| String currentFederateCommitFullyQualifiedVersion = getFederateVersion(project) | |
| if (currentFederateCommitFullyQualifiedVersion == null) { | |
| logger.warn("Ignoring esVersion from `getElasticsearchVersionFromFederateCommit` in `isFederateCommitUsingElasticsearchVersionLessThanOr_6` mostly at build-time.") | |
| return -1 | |
| } | |
| String[] currentFederateCommitVersion = currentFederateCommitFullyQualifiedVersion.split('-') | |
| if (currentFederateCommitVersion.length == 1) { | |
| throw new GradleException("Federate version '$currentFederateCommitFullyQualifiedVersion' must be the fully qualified version. E.g. 7.6.2-20.2, 8.0.0-24.0, etc.") | |
| } | |
| new Semver(currentFederateCommitVersion[1], Semver.SemverType.LOOSE) <=> new Semver(federateVersion, Semver.SemverType.LOOSE) | |
| } | |
| /** | |
| * @param project to get the Federate commit property from. | |
| * @param commit e.g. c280083f661bf16979c5ea8be640ee5b41877906, cd4de5, etc | |
| * @return -1 if Federate commit is chronologically before than specified commit, 0 if the same and 1 if after. | |
| */ | |
| static int compareFederateCommitToCommit(Project project, String commit) { | |
| String federateCommit = getFederateCommit(project) | |
| getDatetimeOfCommit(federateCommit) <=> getDatetimeOfCommit(commit) | |
| } | |
| private static ZonedDateTime getDatetimeOfCommit(String aCommit) { | |
| String hostname = 'api.github.com' | |
| int port = 443 | |
| String scheme = 'https' | |
| HttpHost target = new HttpHost(hostname, port, scheme) | |
| String gitUrl = "$target/repos/sirensolutions/siren-platform/commits/$aCommit".toString() | |
| HttpGet request = new HttpGet(gitUrl) | |
| println("Processing GitHub request ${gitUrl}") | |
| // This method downloads the URI path from GitHub for a particular Git commit if it does not exist. | |
| // The GitHub API is used with the specified GitHub token. | |
| Either<String, String> eitherRequest = HttpClient.executeHttpRequest(target, request, true) | |
| if (eitherRequest.isLeft()) { | |
| throw new GradleException(eitherRequest.getLeft()) | |
| } | |
| String jsonString = eitherRequest.get() | |
| def json = new JsonSlurper().parseText(jsonString) | |
| if (json.commit.committer.date == null) { | |
| throw new GradleException("Failed to download content from $gitUrl\nOutput: ${jsonString}") | |
| } | |
| try { | |
| return ZonedDateTime.parse(json.commit.committer.date.toString()) | |
| } | |
| catch (Exception ex) { | |
| throw new GradleException("Failed to check datetime for commit ${aCommit}", ex) | |
| } | |
| } | |
| /** | |
| * @param project to get the Elasticsearch commit property from. | |
| * @param esVersion 7.10, 7.17, 8.0, etc. | |
| * @return -1 if Elasticsearch commit version is lower than specified version, 0 if the same and 1 if greater. | |
| */ | |
| static int compareEsCommitVersionToEsVersion(Project project, String esVersion) { | |
| String currentEsVersion = getElasticsearchVersionFromFederateCommit(project, true) | |
| new Semver(currentEsVersion, Semver.SemverType.LOOSE) <=> new Semver(esVersion, Semver.SemverType.LOOSE) | |
| } | |
| /** | |
| * See https://sirensolutions.atlassian.net/browse/FEDE-4801 | |
| * @return the Elasticsearch version that the Kubernetes Federate cluster uses. | |
| */ | |
| static @Nullable | |
| String getElasticsearchVersionFromKubernetesConfig(Project project) { | |
| // The benchmark generates the Kubernetes configuration and therefore uses the federate.commit | |
| // to find out about ES version. | |
| getElasticsearchVersionFromFederateCommit(project) | |
| } | |
| /** | |
| * Returns the elasticsearch version of the Federate commit passed as a Gradle property. | |
| * This method does not throw an exception, it may return null. | |
| * | |
| * @param project Project | |
| * @param fail force a fail if cannot get the POM using the project; default to false | |
| * @see BenchmarkConstants#PROP_FEDERATE_COMMIT | |
| * @return Elasticsearch version or null. | |
| */ | |
| static @Nullable | |
| String getElasticsearchVersionFromFederateCommit(Project project, boolean fail = false) { | |
| GPathResult pom | |
| try { | |
| pom = getPomOrDownloadItIfDoesNotExistUsingFederateCommit(project) | |
| } | |
| catch (Exception ex) { | |
| logger.warn("Ignoring call to `getOrDownloadItIfDoesNotExistUsingFederateCommit` in `getElasticsearchVersionFromFederateCommit` mostly at build-time.\n${ex}") | |
| if (fail) { | |
| throw new RuntimeException(ex) | |
| } | |
| } | |
| String version = null | |
| if (pom != null) { | |
| version = pom.'properties'.'elasticsearch.version' | |
| } | |
| version | |
| } | |
| static @Nullable | |
| String getElasticsearchVersionForContainerFromFederateCommit(Project project, boolean fail = false) { | |
| File theFile | |
| try { | |
| final String filePrefix = 'gradle' | |
| final String fileExtension = 'properties' | |
| final String uriPath = "${filePrefix}.${fileExtension}" | |
| theFile = getOrDownloadItIfDoesNotExistUsingFederateCommit(project, filePrefix, fileExtension, uriPath) | |
| } | |
| catch (Exception ex) { | |
| logger.warn("Ignoring call to `getOrDownloadItIfDoesNotExistUsingFederateCommit` in `getElasticsearchVersionForContainerFromFederateCommit` mostly at build-time.\n${ex}") | |
| if (fail) { | |
| throw new RuntimeException(ex) | |
| } | |
| } | |
| String version = null | |
| if (theFile != null) { | |
| Properties properties = new Properties() | |
| properties.load(new FileReader(theFile)) | |
| // This property was introduced on July 1, 2021 by this PR | |
| // https://github.com/sirensolutions/siren-platform/pull/4997/files#diff-b5a06276719e759fe07dfe6f75d781be5f83d2215179d82bdb195ad035348214 | |
| version = properties.getProperty("elasticsearchDockerVersion") | |
| } | |
| version | |
| } | |
| /** | |
| * Returns the elasticsearch version of the Federate commit passed as a Gradle property. | |
| * This method does not throw an exception, it may return null. | |
| * | |
| * @param project Project | |
| * @see BenchmarkConstants#PROP_FEDERATE_COMMIT | |
| * @return Elasticsearch version or throw an exception if fail. | |
| */ | |
| static @Nonnull | |
| String getElasticsearchVersionFromCluster(Project project) { | |
| if (isDryRun(project)) { | |
| 'dry-run-federate-version' | |
| } | |
| else { | |
| Closure cl = { hostname, port -> | |
| String gitUrl = "http://$hostname:$port".toString() | |
| def jsonString = HttpClients.createMinimal().withCloseable { | |
| HttpGet get = new HttpGet(gitUrl) | |
| HttpResponse response = it.execute(get) | |
| EntityUtils.toString(response.getEntity()) | |
| } | |
| def json = new JsonSlurper().parseText(jsonString) | |
| json.'version'.'number' | |
| } | |
| if (isUsingAnExistingCluster(project)) { | |
| def aNode = getNodes(project).iterator().next().split(':') | |
| cl.call aNode[0], aNode[1] | |
| } | |
| else { | |
| (project.spawnCluster as ClusterTask).clusterInfo.get().withFirstNode cl | |
| } | |
| } | |
| } | |
| /** | |
| * Retrieves the Federate version using the Federate commit project property. | |
| * This method does not throw an exception, it may return null. | |
| * | |
| * @param project Project | |
| * @param fail force a fail if cannot get the POM using the project; default to false | |
| * @return Federate version or null. | |
| */ | |
| static @Nullable | |
| String getFederateVersion(Project project, boolean fail = false) { | |
| GPathResult pom | |
| try { | |
| pom = getPomOrDownloadItIfDoesNotExistUsingFederateCommit(project) | |
| } | |
| catch (Exception ex) { | |
| logger.warn("Ignoring call to `getPomOrDownloadItIfDoesNotExistUsingFederateCommit` in `getFederateVersion` mostly at build-time.\n${ex.getMessage()}") | |
| if (fail) { | |
| throw new RuntimeException(ex) | |
| } | |
| } | |
| String version = null | |
| if (pom != null) { | |
| version = pom.'version' | |
| } | |
| version | |
| } | |
| static Map<String, String> getResultPropertyToValue(Project project) { | |
| //// Finally override metrics property based on command line Gradle properties with prefix `results.property.` | |
| //// See JMeterTestCaseProperties.GRADLE_PROPERTY_RESULTS_PREFIX. | |
| //// See benchmark/README.md file for more information on how to pass them. | |
| project.getProperties() | |
| .findAll { | |
| it.getKey().startsWith(GRADLE_PROPERTY_RESULTS_PREFIX) | |
| } | |
| .collectEntries { | |
| [it.getKey().substring(GRADLE_PROPERTY_RESULTS_PREFIX.length()).toLowerCase(), it.getValue()] | |
| } as Map<String, String> | |
| } | |
| private static final String ES_MINIMUM_VERSION_SUPPORTED = '7.10.0' | |
| /** | |
| * This method returns the GCloud Storage information about the snapshot that best fits the project. | |
| * @param project | |
| * @param separator | |
| * @return | |
| */ | |
| static String getGcloudStorageSnapshotInfo(Project project, String separator) { | |
| final Optional<String> datasetName = getDatasetName(project, true) | |
| String bucket = getGcloudSnapshotBucket(project) | |
| String path = datasetName.get() | |
| String name = getSnapshotName(project).get() | |
| String.format("%s%s%s%s%s", bucket, separator, path, separator, name) | |
| } | |
| /** | |
| * This method supersedes the {@link DatasetType#PARENT_CHILD} dataset name to allow using the | |
| * new dataset API for new datasets and to support existing code. | |
| * | |
| * The dataset name returned will be converted to '_' separated lower case words in place of '-' from the | |
| * specified Gradle property {@link #PROP_DATASET_NAME}. | |
| * <p/> | |
| * See <a href="https://sirensolutions.atlassian.net/browse/FEDE-6861">https://sirensolutions.atlassian.net/browse/FEDE-6861</a>. | |
| */ | |
| static Optional<String> getDatasetName(Project project, boolean failIfNotPresent) { | |
| Optional<String> value = null | |
| getGradlePropertyValue( | |
| project, | |
| PROP_DATASET_NAME, | |
| { -> Optional.empty() }, | |
| new, // Any passed option is valid as a dataset name | |
| { String x -> Optional.of(x.toLowerCase().replaceAll('-', '_')) } | |
| ) | |
| if (failIfNotPresent) { | |
| Preconditions.checkArgument(value.isPresent(), | |
| String.format("The name of a dataset name must be provided via a Gradle property called '%s'", PROP_DATASET_NAME) | |
| ) | |
| } | |
| value | |
| } | |
| /** | |
| * This method returns the configuration for a Federate cluster to be deployed on a Google Kubernetes Engine cluster. | |
| */ | |
| static Map<String, String> getKubernetesConfig(Project project) { | |
| // To support creating a snapshot, a Federate cluster must be created first on a Kubernetes cluster with no dataset. | |
| // Therefore the snapshot may not exist. | |
| Optional<String> gcloudSnapshot = getSnapshotName(project) | |
| String nodeCount = 1//DefaultGcloudConfig.getNumberOfNodes() | |
| // Depending on whether a Kubernetes cluster is used or not. | |
| final String clusterId = getContext(project).currentClusterId | |
| final String kubernetesClusterName = getContext(project).kubernetesClusterName | |
| String gcloudServiceAccountFile = project.property(GCS_SERVICE_ACCOUNT_FILE_PROPERTY).toString() | |
| String elasticsearchVersion = getElasticsearchVersionFromFederateCommit(project, true) | |
| String elasticsearchVersionForContainer = getElasticsearchVersionForContainerFromFederateCommit(project, true) | |
| if (elasticsearchVersionForContainer == null) { | |
| // Keep backward compatibility with old fashion style. | |
| elasticsearchVersionForContainer = elasticsearchVersion | |
| } | |
| String elasticsearchDockerImageId = "docker.elastic.co/elasticsearch/elasticsearch:$elasticsearchVersionForContainer" | |
| String elasticsearchMajorVersion = elasticsearchVersion.split('\\.')[0] | |
| final String cidrBlock = getGradlePropertyValue( | |
| project, | |
| PROP_CIDR_BLOCK, | |
| { -> '172.16.0.0/28' }, | |
| { -> }, | |
| { x -> x } | |
| ) | |
| KubernetesConfig.getConfig( | |
| KUBERNETES_ZONE, | |
| KUBERNETES_PROJECT, | |
| clusterId, | |
| gcloudServiceAccountFile, | |
| "projects/$KUBERNETES_PROJECT/global/networks/nat".toString(), | |
| cidrBlock, | |
| 'n1-highmem-8', 'Intel Skylake', | |
| elasticsearchDockerImageId, | |
| elasticsearchMajorVersion, | |
| getFederateCommit(project, true), | |
| getFederateVersion(project, true), | |
| nodeCount, | |
| '7', '40', "$SIREN_MEMORY_ROOT_LIMIT_DATA", '16', | |
| '2', '4', "$SIREN_MEMORY_ROOT_LIMIT_MASTER", '1', | |
| getGcloudSnapshotBucket(project), getDatasetName(project, true).get(), gcloudSnapshot.get(), | |
| getNamespace(project), | |
| getBenchmarkUuid(project), | |
| kubernetesClusterName | |
| ) | |
| } | |
| static Optional<String> getSnapshotName(Project project) { | |
| Optional<String> gcloudSnapshot | |
| final boolean getSnapshotName = getGradlePropertyValue( | |
| project, | |
| 'k8s.env.with.snapshot', | |
| { -> false }, | |
| { -> }, | |
| { x -> x.isEmpty() ? true : Boolean.valueOf(x) } | |
| ) | |
| if (getSnapshotName) { | |
| gcloudSnapshot = getSnapshot(project) | |
| if (!gcloudSnapshot.isPresent()) { | |
| final String msg = "There is no snapshot available for Federate version '${getFederateVersion(project)}'. " + | |
| "Note that '$ES_MINIMUM_VERSION_SUPPORTED' is the minimum Elasticsearch version Federate supports." | |
| throw new IllegalStateException(msg) | |
| } | |
| } | |
| else { | |
| gcloudSnapshot = Optional.of('NO_GCLOUD_SNAPSHOT_DEFINED') | |
| } | |
| return gcloudSnapshot | |
| } | |
| static Optional<String> getSnapshot(Project project) { | |
| FederateFullyQualifiedVersion federateVersion = FederateFullyQualifiedVersion.of(getFederateVersion(project)).orElseThrow() | |
| String bucketName = getGcloudSnapshotBucket(project) | |
| List<String> snapshotsList = getGcpSnapshotsNames(project, bucketName, federateVersion.elasticsearchVersion) | |
| return getSnapshot(federateVersion, snapshotsList, snapshotNameExtension(project)) | |
| } | |
| static String getGcloudSnapshotBucket(Project project) { | |
| String bucketName | |
| switch (getContext(project).executionEnvironment) { | |
| case PRODUCTION: | |
| bucketName = 'tmp-benchmark-snapshots' | |
| break | |
| case RELEASE_CANDIDATE: | |
| case DEVELOPMENT: | |
| case CUSTOM: | |
| default: | |
| bucketName = 'dev-benchmark-snapshots' | |
| } | |
| return bucketName | |
| } | |
| static Optional<String> snapshotNameExtension(Project project) { | |
| getGradlePropertyValue( | |
| project, | |
| PROP_SNAPSHOT_NAME_EXTENSION, | |
| { -> Optional.empty() }, | |
| { -> }, // Any passed option is valid as a dataset name | |
| { String x -> Optional.of(x) } | |
| ) | |
| } | |
| static Optional<String> getSnapshot(FederateFullyQualifiedVersion federateVersion, List<String> snapshotsList, Optional<String> nameExt) { | |
| if (federateVersion.elasticsearchVersion >= new Semver(ES_MINIMUM_VERSION_SUPPORTED, Semver.SemverType.LOOSE)) { | |
| List<SnapshotName> snapshotNames = snapshotsList | |
| .collect(name -> SnapshotName.of(name)) | |
| .findAll(sv -> sv.isPresent()) | |
| .collect(a -> a.get()) | |
| if (nameExt.isPresent()) { | |
| snapshotNames = snapshotNames.findAll { it.name.contains nameExt.get() } | |
| } | |
| snapshotNames.sort (first, second) -> second <=> first // newer on top | |
| for (SnapshotName snapshotName : snapshotNames) { | |
| if (acceptSnapshotVersion(federateVersion, snapshotName)) { | |
| logger.info("Elasticsearch snapshot: ${snapshotName.name} for Federate version ${federateVersion}.") | |
| return Optional.of(snapshotName.name) | |
| } | |
| } | |
| return Optional.empty() | |
| } | |
| else { | |
| throw new GradleException("ES version ${federateVersion.elasticsearchVersion} is not supported. " + | |
| "The minimum version supported is ${ES_MINIMUM_VERSION_SUPPORTED}.") | |
| } | |
| } | |
| private static boolean acceptSnapshotVersion(FederateFullyQualifiedVersion federateVersion, SnapshotName snapshotName) { | |
| Semver.VersionDiff diff = federateVersion.federateVersion.diff(snapshotName.federateVersion) | |
| if (diff == Semver.VersionDiff.MAJOR || snapshotName.federateVersion.isGreaterThan(federateVersion.federateVersion)) { | |
| // not accepted if federate version difference is on Major, or snapshot is greater than federate version | |
| return false | |
| } | |
| // snapshot version should be lower or equal than federate version | |
| return (snapshotName.elasticsearchVersion.isLowerThanOrEqualTo(federateVersion.elasticsearchVersion)) | |
| } | |
| /** | |
| * @param projectId : the ID of your GCP project | |
| * @param bucketName : the ID of your GCS bucket | |
| * @param elasticsearchVersion : the Elasticserach version to use to filter the snapshot objects | |
| */ | |
| static List<String> getGcpSnapshotsNames(Project project, String bucketName, Semver elasticsearchVersion) { | |
| // Disable logging GCloud standard output on console which produces lots of messages | |
| // https://github.com/googleapis/google-http-java-client/blob/8f95371cf5681fbc67bd598d74089f38742a1177/google-http-client/src/main/java/com/google/api/client/http/HttpRequest.java#L895 | |
| java.util.logging.Logger.getLogger('com.google.api.client.http.HttpTransport').setLevel(Level.OFF) | |
| Storage storage = getGoogleStorageService(project) | |
| // List the existing snapshots | |
| Page<Blob> blobs = storage.list( | |
| bucketName, | |
| Storage.BlobListOption.pageSize(100), | |
| // This option avoids going recursively into the bucket + prefix object, lowering the cost. | |
| Storage.BlobListOption.currentDirectory(), | |
| Storage.BlobListOption.prefix(getDatasetName(project, true).get() + '/' + elasticsearchVersion.major), | |
| // This limits the amount data transferred back lowering the cost. | |
| Storage.BlobListOption.fields(Storage.BlobField.NAME) | |
| ) | |
| List<String> snapshotsNames = new ArrayList<>() | |
| for (Blob blob : blobs.iterateAll()) { | |
| snapshotsNames.add(blob.name.split('/')[1]) | |
| } | |
| return snapshotsNames | |
| } | |
| static Storage getGoogleStorageService(Project project) { | |
| // The gcs-benchmark service account must be set otherwise gcloud storage api keeps using the account | |
| // from the server where this command executes. | |
| if (!project.hasProperty(GCS_SERVICE_ACCOUNT_FILE_PROPERTY)) { | |
| throw new IllegalStateException("Gradle property -P${GCS_SERVICE_ACCOUNT_FILE_PROPERTY} must be set to the Google Cloud Service file path.") | |
| } | |
| final String credPath = project.property(GCS_SERVICE_ACCOUNT_FILE_PROPERTY).toString() | |
| final Credentials sourceCredentials = ServiceAccountCredentials.fromStream(new FileInputStream(credPath)) | |
| Storage storage = StorageOptions.newBuilder() | |
| .setCredentials(sourceCredentials) | |
| .setProjectId(KUBERNETES_PROJECT) | |
| .build() | |
| .getService() | |
| storage | |
| } | |
| /** | |
| * This method looks at the extension {@link DatasetPluginExtension} for a user defined GCS object name for the snapshot to create. | |
| * If the extension does not define one, a name is generated for the GCS object to store the snapshot. | |
| * @param project | |
| * @return | |
| * TODO: Refactor and move this method into SnapshotName class (https://sirensolutions.atlassian.net/browse/FEDE-6073) | |
| */ | |
| static String getNameToCreateGcsSnapshot(Project project) { | |
| final DatasetPluginExtension extension = project.extensions.findByName(DatasetPlugin.EXTENSION_NAME) as DatasetPluginExtension | |
| if (extension.isSnapshotIdUserDefined()) { | |
| extension.snapshotIdUserDefined | |
| } | |
| else { | |
| final String timestamp = ZonedDateTime.now(ZoneId.of('UTC')).format(DateTimeFormatter.ofPattern('yyyyMMdd_HHmmss')) | |
| DatasetType datasetType = extension.datasetType.get() | |
| final List<String> strings = [getFederateVersion(project)] | |
| if (datasetType == DatasetType.PARENT_CHILD) { | |
| strings.addAll([ | |
| 'pc', | |
| "p${extension.parentChildExtension.numberOfParents}", | |
| "c${extension.parentChildExtension.numberOfChildrenPerParent}", | |
| "ps${extension.numberOfShards}", | |
| "r${extension.numberOfReplicas}" | |
| ]) | |
| } | |
| else { | |
| final String name = getDatasetName(project, true).get().toLowerCase() | |
| final String datasetNameShort = name.substring(0, Math.min(name.length(), 2)) | |
| strings.add(datasetNameShort) | |
| } | |
| strings.add(timestamp) | |
| final Optional<String> nameExt = snapshotNameExtension(project) | |
| if (nameExt.isPresent()) { | |
| strings.add(nameExt.get()) | |
| } | |
| final String snapshotName = strings | |
| .join('_') | |
| .toString() | |
| if (getDatasetName(project, false).isPresent()) { | |
| // This allows to use the new dataset API with the existing orchestrator project. | |
| // See https://sirensolutions.atlassian.net/browse/FEDE-6861 | |
| return snapshotName | |
| } | |
| else { | |
| switch (datasetType) { | |
| case DatasetType.PARENT_CHILD: | |
| return snapshotName | |
| case DatasetType.AWS_S3: | |
| case DatasetType.MOCK: | |
| return "${snapshotName}_${datasetType.id.toLowerCase()}" | |
| case DatasetType.ELASTICDUMP: | |
| return [snapshotName, | |
| "dps${extension.elasticdumpExtension.dateStartDump.format(DateTimeFormatter.ISO_INSTANT)}", | |
| "dpe${extension.elasticdumpExtension.dateEndDump.format(DateTimeFormatter.ISO_INSTANT)}", | |
| extension.elasticdumpExtension.nbOfEntries, | |
| 'elasticdump'] | |
| .join('_') | |
| .toLowerCase() | |
| default: | |
| throw new GradleException("Unable to create a snapshot ID for a ${datasetType} dataset") | |
| } | |
| } | |
| } | |
| } | |
| static String getNamespace(Project project) { | |
| // To comply the namespace must be a lowercase RFC 1123 label with at most 63 characters. | |
| String res = "bk-${getBenchmarkUuid(project).replaceAll('_', '-').toLowerCase()}" | |
| // https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/#namespaces-and-dns | |
| Preconditions.checkArgument(res.length() <= 63, "Kubernetes namespace must be at most 63 characters") | |
| res | |
| } | |
| /* | |
| This part handles gcloud-cicd repository to handle Kubernetes clusters and Federate deployment on it. | |
| See https://sirensolutions.atlassian.net/browse/FEDE-5448 | |
| gcloud-cicd repo can be provided from a local directory. | |
| */ | |
| private boolean isSirenGCloudCicdInstallDirProvided = false | |
| private String sirenGCloudCicdInstallDirBranch = null | |
| private Path sirenGCloudCicdInstallDir = null | |
| /** | |
| * This method checks if cloud tools are installed. It could be from the gcloud command line to the | |
| * repositories Federate team uses to run stuff on Google cloud or Google Kubernetes Engine. | |
| */ | |
| static boolean areCloudToolsInstalled(Project project) { | |
| File f = getContext(project).sirenGCloudCicdInstallDir.toFile() | |
| f.exists() && f.isDirectory() | |
| } | |
| static Path getSirenGCloudCicdInstallDir(Project project) { | |
| getContext(project).sirenGCloudCicdInstallDir | |
| } | |
| static void reinstallCloudToolsOrFail(Project project) { | |
| final BenchmarkContext context = getContext(project) | |
| if (!context.isSirenGCloudCicdInstallDirProvided) { | |
| File dir = context.sirenGCloudCicdInstallDir.toFile() | |
| if (!dir.deleteDir()) { | |
| throw new RuntimeException("Failed to delete the directory ${dir}.") | |
| } | |
| installCloudToolsOrFail(project) | |
| } | |
| } | |
| private static void forceKillKubernetes(Project project) { | |
| getContext(project).setReusableClusterRunning() | |
| getContext(project).setReusableClusterDelete() | |
| } | |
| /** | |
| * This method can use an existing local directory where gcloud-cicd is or clone gcloud-cicd repository using the | |
| * specified branch name from Gradle property {@link #PROP_KUBERNETES_GIT_BRANCH}. | |
| * <p/> | |
| * See https://sirensolutions.atlassian.net/browse/FEDE-5448 | |
| * @param project | |
| */ | |
| static void installCloudToolsOrFail(Project project) { | |
| final BenchmarkContext context = getContext(project) | |
| final File dir = context.sirenGCloudCicdInstallDir.toFile() | |
| if (!context.isSirenGCloudCicdInstallDirProvided && !dir.exists()) { | |
| assert context.sirenGCloudCicdInstallDirBranch != null: 'Git branch must be set in "context.sirenGCloudCicdInstallDirBranch"' | |
| if (areCloudToolsInstalled(project)) { | |
| throw new IllegalStateException("GCloud CICD is already initialized.\n It cannot be initialized more than once.") | |
| } | |
| if (!dir.mkdirs()) { | |
| throw new RuntimeException("Failed to create the directory ${dir}.") | |
| } | |
| final String gitRepo = 'github.com/sirensolutions/gcloud-cicd.git' | |
| final String commandToExecute = "git clone https://${gitRepo} ${dir}" | |
| ProcessUtils.processBashCommand(commandToExecute, true) | |
| assert areCloudToolsInstalled(project): 'installCloudToolsOrFail(...) failed to clone gclouc-cicd from Git repo ${gitRepo}' | |
| ProcessBuilder pb = new ProcessBuilder('git', 'pull', '-f') | |
| pb.directory(context.sirenGCloudCicdInstallDir.toFile()) | |
| Process p = pb.start() | |
| ProcessUtils.processBashCommand(p, true) | |
| pb = new ProcessBuilder('git', 'checkout', context.sirenGCloudCicdInstallDirBranch) | |
| pb.directory(context.sirenGCloudCicdInstallDir.toFile()) | |
| p = pb.start() | |
| ProcessUtils.processBashCommand(p, true) | |
| logger.info("Changed gcloud cicd repo to branch ${context.sirenGCloudCicdInstallDirBranch}") | |
| } | |
| } | |
| /** | |
| * A benchmark can run only on one cluster. User can decide to use a custom cluster or a kubernetes cluster using 2 different parameters | |
| * that are exclusives. | |
| * This methods checks only one type of cluster is specified through one of the parameters. | |
| * @see BenchmarkContext#PROP_CLUSTER_CUSTOM | |
| * @see BenchmarkContext#PROP_CLUSTER_KUBERNETES | |
| */ | |
| static void checkOnlyOneClusterIsRequestedOrFail(Project project) { | |
| if (isUsingAnExistingCluster(project) && isSpawningAKubernetesCluster(project)) { | |
| errorMultipleClusterTypeAndFail(project) | |
| } | |
| } | |
| static Optional<Integer> getKubernetesCmdPid(Project project) { | |
| Optional.ofNullable(getContext(project).clusterPid == null ? null : getContext(project).clusterPid) | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment