Skip to content

Instantly share code, notes, and snippets.

@tsibley

tsibley/pathogen-repo-builders.yaml

Created February 12, 2024 19:04
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save tsibley/2402348bcef12edab76b94413c2a3908 to your computer and use it in GitHub Desktop.
Save tsibley/2402348bcef12edab76b94413c2a3908 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
pathogen-repo-builders.yaml
- workflow: ./mpox/.github/workflows/fetch-and-ingest-branch.yaml
jobs:
fetch-and-ingest:
needs: [set_config_overrides]
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
env: |
NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }}
GITHUB_RUN_ID: ${{ github.run_id }}
SLACK_CHANNELS: ${{ vars.TEST_SLACK_CHANNEL }}
UPLOAD_CONFIG: ${{ needs.set_config_overrides.outputs.upload_config }}
run: |
nextstrain build \
--aws-batch \
--detach \
--no-download \
--cpus 32 \
--memory 64gib \
--env AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY \
--env GITHUB_RUN_ID \
--env SLACK_TOKEN \
--env SLACK_CHANNELS \
ingest \
nextstrain_automation \
--configfiles build-configs/nextstrain-automation/config.yaml \
--config trigger_rebuild=False send_slack_notifications=True upload="$UPLOAD_CONFIG"
- workflow: ./mpox/.github/workflows/ci.yaml
jobs:
pathogen-ci:
strategy:
matrix:
runtime: [docker, conda]
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: ${{ matrix.runtime }}
run: |
nextstrain build \
phylogenetic \
--configfile profiles/ci/builds.yaml
artifact-name: output-${{ matrix.runtime }}
artifact-paths: |
phylogenetic/auspice/
phylogenetic/results/
phylogenetic/benchmarks/
phylogenetic/logs/
phylogenetic/.snakemake/log/
- workflow: ./mpox/.github/workflows/fetch-and-ingest.yaml
jobs:
fetch-and-ingest:
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
env: |
NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }}
GITHUB_RUN_ID: ${{ github.run_id }}
SLACK_CHANNELS: ${{ vars.SLACK_CHANNELS }}
run: |
nextstrain build \
--detach \
--no-download \
--cpus 32 \
--memory 64gib \
--env AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY \
--env GITHUB_RUN_ID \
--env SLACK_TOKEN \
--env SLACK_CHANNELS \
--env PAT_GITHUB_DISPATCH="$GH_TOKEN_NEXTSTRAIN_BOT_WORKFLOW_DISPATCH" \
ingest \
nextstrain_automation \
--configfiles build-configs/nextstrain-automation/config.yaml \
- workflow: ./mpox/.github/workflows/rebuild-hmpxv1-big.yaml
jobs:
rebuild_hmpxv1_big:
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
env: |
NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }}
GITHUB_RUN_ID: ${{ github.run_id }}
SLACK_CHANNELS: ${{ vars.SLACK_CHANNELS }}
BUILD_DIR: phylogenetic
BUILD_NAME: hmpxv1_big
run: |
nextstrain build \
--detach \
--no-download \
--cpus 8 \
--memory 68gib \
--env AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY \
--env GITHUB_RUN_ID \
--env SLACK_TOKEN \
--env SLACK_CHANNELS \
. \
notify_on_deploy \
--configfiles $BUILD_DIR/config/$BUILD_NAME/config.yaml $BUILD_DIR/config/nextstrain_automation.yaml \
--config auspice_prefix=$TRIAL_NAME --directory $BUILD_DIR --snakefile $BUILD_DIR/Snakefile
- workflow: ./mpox/.github/workflows/rebuild-mpxv.yaml
jobs:
rebuild_mpxv:
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
env: |
NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }}
GITHUB_RUN_ID: ${{ github.run_id }}
SLACK_CHANNELS: ${{ vars.SLACK_CHANNELS }}
BUILD_DIR: phylogenetic
BUILD_NAME: mpxv
run: |
nextstrain build \
--detach \
--no-download \
--cpus 8 \
--memory 32gib \
--env AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY \
--env GITHUB_RUN_ID \
--env SLACK_TOKEN \
--env SLACK_CHANNELS \
. \
notify_on_deploy \
--configfiles $BUILD_DIR/config/$BUILD_NAME/config.yaml $BUILD_DIR/config/nextstrain_automation.yaml \
--config auspice_prefix=$TRIAL_NAME --directory $BUILD_DIR --snakefile $BUILD_DIR/Snakefile
- workflow: ./mpox/.github/workflows/rebuild-hmpxv1.yaml
jobs:
rebuild_hmpxv1:
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
env: |
NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.image }}
GITHUB_RUN_ID: ${{ github.run_id }}
SLACK_CHANNELS: ${{ vars.SLACK_CHANNELS }}
BUILD_DIR: phylogenetic
BUILD_NAME: hmpxv1
run: |
nextstrain build \
--detach \
--no-download \
--cpus 8 \
--memory 32gib \
--env AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY \
--env GITHUB_RUN_ID \
--env SLACK_TOKEN \
--env SLACK_CHANNELS \
. \
notify_on_deploy \
--configfiles $BUILD_DIR/config/$BUILD_NAME/config.yaml $BUILD_DIR/config/nextstrain_automation.yaml \
--config auspice_prefix=$TRIAL_NAME --directory $BUILD_DIR --snakefile $BUILD_DIR/Snakefile
- workflow: ./forecasts-ncov/.github/workflows/run-models.yaml
jobs:
run_models:
needs: [set_config_overrides]
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
run: |
nextstrain build \
--aws-batch \
--detach \
--no-download \
--image nextstrain/base \
--cpus 8 \
--memory 16GiB \
--env AWS_DEFAULT_REGION \
--env AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY \
--env GITHUB_RUN_ID \
--env SLACK_TOKEN \
--env SLACK_CHANNELS \
. \
--configfile config/config.yaml config/optional.yaml \
--config s3_dst="$S3_DST" data_provenances="$DATA_PROVENANCES" \
--keep-going
env: |
DATA_PROVENANCES: ${{ needs.set_config_overrides.outputs.data_provenances }}
GITHUB_RUN_ID: ${{ github.run_id }}
SLACK_CHANNELS: ${{ github.event.inputs.slack_channel || vars.SLACK_CHANNELS }}
S3_DST: ${{ needs.set_config_overrides.outputs.s3_dst }}
- workflow: ./forecasts-ncov/.github/workflows/update-ncov-open-clade-counts.yaml
jobs:
open_clade_counts:
needs: [set_s3_dst]
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
run: |
nextstrain build \
--cpus 8 \
--memory 16gib \
--env AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY \
--env SLACK_TOKEN \
--env SLACK_CHANNELS \
--env S3_DST \
ingest \
upload_all_sequence_counts \
--config data_provenances=["open"] s3_dst="$S3_DST"
env: |
SLACK_CHANNELS: ${{ inputs.slack_channel || vars.SLACK_CHANNELS }}
S3_DST: ${{ needs.set_s3_dst.outputs.s3_dst }}
- workflow: ./forecasts-ncov/.github/workflows/update-ncov-gisaid-clade-counts.yaml
jobs:
gisaid_clade_counts:
needs: [set_s3_dst]
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
run: |
nextstrain build \
--cpus 8 \
--memory 16gib \
--env AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY \
--env SLACK_TOKEN \
--env SLACK_CHANNELS \
--env S3_DST \
ingest \
upload_all_sequence_counts \
--config data_provenances=["gisaid"] s3_dst="$S3_DST"
env: |
SLACK_CHANNELS: ${{ inputs.slack_channel || vars.SLACK_CHANNELS }}
S3_DST: ${{ needs.set_s3_dst.outputs.s3_dst }}
- workflow: ./hepatitisB/.github/workflows/run-builds.yaml
jobs:
run-build:
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: conda
run: |
nextstrain build --cpus 1 . -pf vendor_nextclade3_x86;
nextstrain build --cpus 1 . -pf deploy;
- workflow: ./zika/.github/workflows/ci.yaml
jobs:
pathogen-ci:
strategy:
matrix:
runtime: [docker, conda]
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: ${{ matrix.runtime }}
run: |
nextstrain build \
phylogenetic \
--configfile profiles/ci/profiles_config.yaml
artifact-name: output-${{ matrix.runtime }}
artifact-paths: |-
phylogenetic/auspice/
phylogenetic/results/
phylogenetic/benchmarks/
phylogenetic/logs/
phylogenetic/.snakemake/log/
- workflow: ./seasonal-flu/.github/workflows/run-nextflu-private-builds.yaml
jobs:
run-build:
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
env: |
NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.dockerImage }}
run: |
nextstrain build \
--detach \
--cpus 36 \
--memory 72gib \
--env AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY \
--env NEXTSTRAIN_USERNAME \
--env NEXTSTRAIN_PASSWORD \
. \
deploy_all \
all_counts_of_recent_tips_by_clade \
-p \
--configfile profiles/nextflu-private.yaml
- workflow: ./seasonal-flu/.github/workflows/run-private-nextflu-builds.yaml
jobs:
run-build:
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
env: |
NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.dockerImage }}
run: |
nextstrain build \
--detach \
--cpus 36 \
--memory 72gib \
--env AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY \
. \
all_who \
-p \
--configfile profiles/private.nextflu.org.yaml
- workflow: ./seasonal-flu/.github/workflows/run-public-builds.yaml
jobs:
run-build:
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
env: |
NEXTSTRAIN_DOCKER_IMAGE: ${{ inputs.dockerImage }}
run: |
nextstrain build \
--detach \
--cpus 36 \
--memory 72gib \
--env AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY \
. \
deploy_all \
-p \
--configfile profiles/nextstrain-public.yaml
- workflow: ./seasonal-flu/.github/workflows/upload.yaml
jobs:
upload:
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: docker
run: |
nextstrain build \
--env AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY \
--env RETHINK_AUTH_KEY \
--env RETHINK_HOST \
. \
-j 4 \
upload_all_titers \
upload_all_raw_sequences \
upload_all_sequences \
upload_all_metadata \
--configfile profiles/upload.yaml
- workflow: ./dengue/.github/workflows/ci.yaml
jobs:
pathogen-ci:
strategy:
matrix:
runtime: [docker, conda]
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: ${{ matrix.runtime }}
run: |
nextstrain build \
phylogenetic \
--configfile profiles/ci/profiles_config.yaml
artifact-name: output-${{ matrix.runtime }}
artifact-paths: |-
phylogenetic/auspice/
phylogenetic/results/
phylogenetic/benchmarks/
phylogenetic/logs/
phylogenetic/.snakemake/log/
- workflow: ./.github/workflow-templates/pathogen-repo-build.yaml
jobs:
run-build:
# The `id-token: write` permission is required for this workflow.
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
- workflow: ./.github/.github/workflows/ci.yaml
jobs:
test-pathogen-repo-build:
permissions:
id-token: write
strategy:
matrix:
runtime: [docker, conda, aws-batch]
uses: ./.github/workflows/pathogen-repo-build.yaml
with:
repo: nextstrain/zika-tutorial
runtime: ${{ matrix.runtime }}
run: nextstrain build --env BOGUS_ENVVAR .
artifact-name: build-output-${{ matrix.runtime }}
env: |
BOGUS_ENVVAR: BOGUS_ENVVAR
secrets: inherit
- workflow: ./rsv/.github/workflows/fetch-and-ingest.yaml
jobs:
fetch-and-ingest:
needs: [set_config_overrides]
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
run: |
nextstrain build \
--aws-batch \
--detach \
--no-download \
--cpus 8 \
--memory 32gib \
--env AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY \
--env PAT_GITHUB_DISPATCH="$GH_TOKEN_NEXTSTRAIN_BOT_WORKFLOW_DISPATCH" \
--env S3_DST \
--env TRIGGER_REBUILD \
ingest \
--configfiles config/config.yaml config/optional.yaml \
--config s3_dst="$S3_DST" trigger_rebuild="$TRIGGER_REBUILD" \
--printshellcmds
env: |
S3_DST: ${{ needs.set_config_overrides.outputs.s3_dst }}
TRIGGER_REBUILD: ${{ needs.set_config_overrides.outputs.trigger_rebuild }}
- workflow: ./rsv/.github/workflows/rebuild.yaml
jobs:
rebuild_rsv:
permissions:
id-token: write
uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
secrets: inherit
with:
runtime: aws-batch
run: |
nextstrain build \
--aws-batch \
--detach \
--no-download \
--cpus 16 \
--memory 64gib \
--env AWS_ACCESS_KEY_ID \
--env AWS_SECRET_ACCESS_KEY \
. \
deploy \
--configfiles config/configfile.yaml config/nextstrain_automation.yaml \
--printshellcmds
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment