Created
April 26, 2024 22:59
-
-
Save enakai00/ab539e9401ed9fc2ad5fceacb164003e to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "components": { | |
| "comp-compose-params-for-model-tuner": { | |
| "executorLabel": "exec-compose-params-for-model-tuner", | |
| "inputDefinitions": { | |
| "parameters": { | |
| "adapter_length": { | |
| "defaultValue": 0, | |
| "description": "Adapter length.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_INTEGER" | |
| }, | |
| "display_name": { | |
| "description": "The display name of the model tuner.", | |
| "parameterType": "STRING" | |
| }, | |
| "enable_checkpoint_selection": { | |
| "defaultValue": "default", | |
| "description": "One of 'true', 'false', or 'default'. If set\nto 'true' and if evaluation_data_uri is set, select the best tuned\ncheckpoint. If 'default' is set, checkpoint selection is enabled\nfor bison@001 models, and disabled for others.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "enable_early_stopping": { | |
| "defaultValue": true, | |
| "description": "If set to True, and if evaluation_data_uri is set,\nthen early stopping will be enabled. If early stopping is enabled, the\nmax recommended train_steps is 1,000.", | |
| "isOptional": true, | |
| "parameterType": "BOOLEAN" | |
| }, | |
| "encryption_spec_key_name": { | |
| "defaultValue": "", | |
| "description": "Customer-managed encryption key. If this is set,\nthen all resources created by the CustomJob will be encrypted with the\nprovided encryption key. Note that this is not supported for TPU at\nthe moment.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "evaluation_data_uri": { | |
| "defaultValue": "", | |
| "description": "GCS URI of eval dataset.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "evaluation_interval": { | |
| "defaultValue": 20, | |
| "description": "Evaluation after every this number of tuning steps.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_INTEGER" | |
| }, | |
| "learning_rate": { | |
| "defaultValue": -1, | |
| "description": "Learning rate hyperparameter. When learning rate is less\nthan 0, it uses the default learning rate per tuning method per model.\nThis parameter will be deprecated.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_DOUBLE" | |
| }, | |
| "learning_rate_multiplier": { | |
| "defaultValue": 1, | |
| "description": "Learning rate multiplier for tuning. Under the\nhood, the actual \"learning rate\" will be learning_rate_multiplier *\nthe \"recommended learning rate\" per model, so customers don't have to\nkeep track of the recommended learning rate for each model or tuning\nmethod.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_DOUBLE" | |
| }, | |
| "location": { | |
| "defaultValue": "us-central1", | |
| "description": "Region in which the launcher for the tuning job is created (not\nthe region for running the whole pipeline).", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "max_context_length": { | |
| "defaultValue": "", | |
| "description": "The max context length used for tuning. Can be either\n'8k' or '32k'.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "model": { | |
| "defaultValue": "text-bison@001", | |
| "description": "Model type to use.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "pipeline_region": { | |
| "defaultValue": "{{$.pipeline_google_cloud_location}}", | |
| "description": "The region the pipeline runs in.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "tpu_training_skip_cmek": { | |
| "defaultValue": false, | |
| "description": "If True, skip CMEK setup for TPU training.", | |
| "isOptional": true, | |
| "parameterType": "BOOLEAN" | |
| }, | |
| "train_data_list": { | |
| "description": "List of dict representation of training_data\nwith its key restricted to `path` and `weight`.", | |
| "isOptional": true, | |
| "parameterType": "LIST" | |
| }, | |
| "train_data_uri": { | |
| "description": "File or file pattern containing train data.", | |
| "parameterType": "STRING" | |
| }, | |
| "train_data_validation_result": { | |
| "description": "Json object containing validation result of\nthe training dataset.", | |
| "isOptional": true, | |
| "parameterType": "STRUCT" | |
| }, | |
| "train_steps": { | |
| "defaultValue": 0, | |
| "description": "Number of training steps.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_INTEGER" | |
| }, | |
| "tuning_method": { | |
| "defaultValue": "tune_v2", | |
| "description": "The model tuning method. Available methods: tune,\ntune_v2.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "use_tpu": { | |
| "defaultValue": true, | |
| "description": "If True, use TPU. Otherwise use GPU.", | |
| "isOptional": true, | |
| "parameterType": "BOOLEAN" | |
| } | |
| } | |
| }, | |
| "outputDefinitions": { | |
| "parameters": { | |
| "location": { | |
| "parameterType": "STRING" | |
| }, | |
| "payload_first_part": { | |
| "parameterType": "STRING" | |
| }, | |
| "payload_second_part": { | |
| "parameterType": "STRING" | |
| } | |
| } | |
| } | |
| }, | |
| "comp-condition-1": { | |
| "dag": { | |
| "tasks": { | |
| "tensorboard-uploader": { | |
| "cachingOptions": { | |
| "enableCache": true | |
| }, | |
| "componentRef": { | |
| "name": "comp-tensorboard-uploader" | |
| }, | |
| "inputs": { | |
| "artifacts": { | |
| "metrics_directory_uri": { | |
| "componentInputArtifact": "pipelinechannel--tuning-graph-tensorboard_metrics" | |
| } | |
| }, | |
| "parameters": { | |
| "experiment_name": { | |
| "runtimeValue": { | |
| "constant": "large-language-model-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}" | |
| } | |
| }, | |
| "location": { | |
| "runtimeValue": { | |
| "constant": "{{$.pipeline_google_cloud_location}}" | |
| } | |
| }, | |
| "project": { | |
| "componentInputParameter": "pipelinechannel--project" | |
| }, | |
| "tensorboard_id": { | |
| "componentInputParameter": "pipelinechannel--tensorboard_resource_id" | |
| } | |
| } | |
| }, | |
| "taskInfo": { | |
| "name": "tensorboard-uploader" | |
| } | |
| } | |
| } | |
| }, | |
| "inputDefinitions": { | |
| "artifacts": { | |
| "pipelinechannel--tuning-graph-tensorboard_metrics": { | |
| "artifactType": { | |
| "schemaTitle": "system.Artifact", | |
| "schemaVersion": "0.0.1" | |
| } | |
| } | |
| }, | |
| "parameters": { | |
| "pipelinechannel--project": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--tensorboard_resource_id": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--validate-pipeline-with_tensorboard": { | |
| "parameterType": "BOOLEAN" | |
| } | |
| } | |
| } | |
| }, | |
| "comp-condition-2": { | |
| "dag": { | |
| "outputs": { | |
| "parameters": { | |
| "pipelinechannel--dataset-encoder-2-dataset_path": { | |
| "valueFromParameter": { | |
| "outputParameterKey": "dataset_path", | |
| "producerSubtask": "dataset-encoder-2" | |
| } | |
| }, | |
| "pipelinechannel--dataset-encoder-dataset_path": { | |
| "valueFromParameter": { | |
| "outputParameterKey": "dataset_path", | |
| "producerSubtask": "dataset-encoder" | |
| } | |
| } | |
| } | |
| }, | |
| "tasks": { | |
| "dataset-encoder": { | |
| "cachingOptions": {}, | |
| "componentRef": { | |
| "name": "comp-dataset-encoder" | |
| }, | |
| "inputs": { | |
| "parameters": { | |
| "dataset_uri": { | |
| "componentInputParameter": "pipelinechannel--export-managed-dataset-export_location" | |
| }, | |
| "encryption_spec_key_name": { | |
| "componentInputParameter": "pipelinechannel--encryption_spec_key_name" | |
| }, | |
| "large_model_reference": { | |
| "componentInputParameter": "pipelinechannel--model" | |
| }, | |
| "project": { | |
| "componentInputParameter": "pipelinechannel--project" | |
| } | |
| } | |
| }, | |
| "taskInfo": { | |
| "name": "dataset-encoder" | |
| } | |
| }, | |
| "dataset-encoder-2": { | |
| "cachingOptions": {}, | |
| "componentRef": { | |
| "name": "comp-dataset-encoder-2" | |
| }, | |
| "inputs": { | |
| "parameters": { | |
| "dataset_uri": { | |
| "componentInputParameter": "pipelinechannel--evaluation_data_uri" | |
| }, | |
| "encryption_spec_key_name": { | |
| "componentInputParameter": "pipelinechannel--encryption_spec_key_name" | |
| }, | |
| "large_model_reference": { | |
| "componentInputParameter": "pipelinechannel--model" | |
| }, | |
| "max_examples": { | |
| "runtimeValue": { | |
| "constant": 256 | |
| } | |
| }, | |
| "project": { | |
| "componentInputParameter": "pipelinechannel--project" | |
| } | |
| } | |
| }, | |
| "taskInfo": { | |
| "name": "evaluation-dataset-encoder" | |
| } | |
| } | |
| } | |
| }, | |
| "inputDefinitions": { | |
| "parameters": { | |
| "pipelinechannel--encryption_spec_key_name": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--evaluation_data_uri": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--export-managed-dataset-export_location": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--model": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--project": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--use_v2_preprocessing": { | |
| "parameterType": "BOOLEAN" | |
| } | |
| } | |
| }, | |
| "outputDefinitions": { | |
| "parameters": { | |
| "pipelinechannel--dataset-encoder-2-dataset_path": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--dataset-encoder-dataset_path": { | |
| "parameterType": "STRING" | |
| } | |
| } | |
| } | |
| }, | |
| "comp-condition-3": { | |
| "dag": { | |
| "outputs": { | |
| "parameters": { | |
| "pipelinechannel--dataset-encoder-3-dataset_path": { | |
| "valueFromParameter": { | |
| "outputParameterKey": "dataset_path", | |
| "producerSubtask": "dataset-encoder-3" | |
| } | |
| }, | |
| "pipelinechannel--dataset-encoder-4-dataset_path": { | |
| "valueFromParameter": { | |
| "outputParameterKey": "dataset_path", | |
| "producerSubtask": "dataset-encoder-4" | |
| } | |
| }, | |
| "pipelinechannel--dummy-prompt-validation-component-Output": { | |
| "valueFromParameter": { | |
| "outputParameterKey": "Output", | |
| "producerSubtask": "dummy-prompt-validation-component" | |
| } | |
| } | |
| } | |
| }, | |
| "tasks": { | |
| "dataset-encoder-3": { | |
| "cachingOptions": {}, | |
| "componentRef": { | |
| "name": "comp-dataset-encoder-3" | |
| }, | |
| "inputs": { | |
| "parameters": { | |
| "dataset_uri": { | |
| "componentInputParameter": "pipelinechannel--export-managed-dataset-export_location" | |
| }, | |
| "encryption_spec_key_name": { | |
| "componentInputParameter": "pipelinechannel--encryption_spec_key_name" | |
| }, | |
| "image_uri": { | |
| "runtimeValue": { | |
| "constant": "us-docker.pkg.dev/vertex-ai-restricted/genai-tuning-restricted/llm-accessory-experiment:v1.0.0" | |
| } | |
| }, | |
| "large_model_reference": { | |
| "componentInputParameter": "pipelinechannel--model" | |
| }, | |
| "project": { | |
| "componentInputParameter": "pipelinechannel--project" | |
| } | |
| } | |
| }, | |
| "taskInfo": { | |
| "name": "dataset-encoder" | |
| } | |
| }, | |
| "dataset-encoder-4": { | |
| "cachingOptions": {}, | |
| "componentRef": { | |
| "name": "comp-dataset-encoder-4" | |
| }, | |
| "inputs": { | |
| "parameters": { | |
| "dataset_uri": { | |
| "componentInputParameter": "pipelinechannel--evaluation_data_uri" | |
| }, | |
| "encryption_spec_key_name": { | |
| "componentInputParameter": "pipelinechannel--encryption_spec_key_name" | |
| }, | |
| "image_uri": { | |
| "runtimeValue": { | |
| "constant": "us-docker.pkg.dev/vertex-ai-restricted/genai-tuning-restricted/llm-accessory-experiment:v1.0.0" | |
| } | |
| }, | |
| "large_model_reference": { | |
| "componentInputParameter": "pipelinechannel--model" | |
| }, | |
| "max_examples": { | |
| "runtimeValue": { | |
| "constant": 256 | |
| } | |
| }, | |
| "project": { | |
| "componentInputParameter": "pipelinechannel--project" | |
| } | |
| } | |
| }, | |
| "taskInfo": { | |
| "name": "evaluation-dataset-encoder" | |
| } | |
| }, | |
| "dummy-prompt-validation-component": { | |
| "cachingOptions": { | |
| "enableCache": true | |
| }, | |
| "componentRef": { | |
| "name": "comp-dummy-prompt-validation-component" | |
| }, | |
| "dependentTasks": [ | |
| "dataset-encoder-3" | |
| ], | |
| "inputs": { | |
| "parameters": { | |
| "encoded_dataset_stats": { | |
| "taskOutputParameter": { | |
| "outputParameterKey": "encoded_dataset_stats", | |
| "producerTask": "dataset-encoder-3" | |
| } | |
| } | |
| } | |
| }, | |
| "taskInfo": { | |
| "name": "dummy-prompt-validation-component" | |
| } | |
| } | |
| } | |
| }, | |
| "inputDefinitions": { | |
| "parameters": { | |
| "pipelinechannel--encryption_spec_key_name": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--evaluation_data_uri": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--export-managed-dataset-export_location": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--model": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--project": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--use_v2_preprocessing": { | |
| "parameterType": "BOOLEAN" | |
| } | |
| } | |
| }, | |
| "outputDefinitions": { | |
| "parameters": { | |
| "pipelinechannel--dataset-encoder-3-dataset_path": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--dataset-encoder-4-dataset_path": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--dummy-prompt-validation-component-Output": { | |
| "parameterType": "STRUCT" | |
| } | |
| } | |
| } | |
| }, | |
| "comp-condition-branches-1": { | |
| "dag": { | |
| "outputs": { | |
| "parameters": { | |
| "pipelinechannel--condition-branches-1-oneof-1": { | |
| "valueFromOneof": { | |
| "parameterSelectors": [ | |
| { | |
| "outputParameterKey": "pipelinechannel--dataset-encoder-dataset_path", | |
| "producerSubtask": "condition-2" | |
| }, | |
| { | |
| "outputParameterKey": "pipelinechannel--dataset-encoder-3-dataset_path", | |
| "producerSubtask": "condition-3" | |
| } | |
| ] | |
| } | |
| }, | |
| "pipelinechannel--condition-branches-1-oneof-2": { | |
| "valueFromOneof": { | |
| "parameterSelectors": [ | |
| { | |
| "outputParameterKey": "pipelinechannel--dataset-encoder-2-dataset_path", | |
| "producerSubtask": "condition-2" | |
| }, | |
| { | |
| "outputParameterKey": "pipelinechannel--dataset-encoder-4-dataset_path", | |
| "producerSubtask": "condition-3" | |
| } | |
| ] | |
| } | |
| }, | |
| "pipelinechannel--condition-branches-1-oneof-3": { | |
| "valueFromOneof": { | |
| "parameterSelectors": [ | |
| { | |
| "outputParameterKey": "pipelinechannel--dummy-prompt-validation-component-Output", | |
| "producerSubtask": "condition-3" | |
| } | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| "tasks": { | |
| "condition-2": { | |
| "componentRef": { | |
| "name": "comp-condition-2" | |
| }, | |
| "inputs": { | |
| "parameters": { | |
| "pipelinechannel--encryption_spec_key_name": { | |
| "componentInputParameter": "pipelinechannel--encryption_spec_key_name" | |
| }, | |
| "pipelinechannel--evaluation_data_uri": { | |
| "componentInputParameter": "pipelinechannel--evaluation_data_uri" | |
| }, | |
| "pipelinechannel--export-managed-dataset-export_location": { | |
| "componentInputParameter": "pipelinechannel--export-managed-dataset-export_location" | |
| }, | |
| "pipelinechannel--model": { | |
| "componentInputParameter": "pipelinechannel--model" | |
| }, | |
| "pipelinechannel--project": { | |
| "componentInputParameter": "pipelinechannel--project" | |
| }, | |
| "pipelinechannel--use_v2_preprocessing": { | |
| "componentInputParameter": "pipelinechannel--use_v2_preprocessing" | |
| } | |
| } | |
| }, | |
| "taskInfo": { | |
| "name": "preprocess_v1" | |
| }, | |
| "triggerPolicy": { | |
| "condition": "inputs.parameter_values['pipelinechannel--use_v2_preprocessing'] == false" | |
| } | |
| }, | |
| "condition-3": { | |
| "componentRef": { | |
| "name": "comp-condition-3" | |
| }, | |
| "inputs": { | |
| "parameters": { | |
| "pipelinechannel--encryption_spec_key_name": { | |
| "componentInputParameter": "pipelinechannel--encryption_spec_key_name" | |
| }, | |
| "pipelinechannel--evaluation_data_uri": { | |
| "componentInputParameter": "pipelinechannel--evaluation_data_uri" | |
| }, | |
| "pipelinechannel--export-managed-dataset-export_location": { | |
| "componentInputParameter": "pipelinechannel--export-managed-dataset-export_location" | |
| }, | |
| "pipelinechannel--model": { | |
| "componentInputParameter": "pipelinechannel--model" | |
| }, | |
| "pipelinechannel--project": { | |
| "componentInputParameter": "pipelinechannel--project" | |
| }, | |
| "pipelinechannel--use_v2_preprocessing": { | |
| "componentInputParameter": "pipelinechannel--use_v2_preprocessing" | |
| } | |
| } | |
| }, | |
| "taskInfo": { | |
| "name": "preprocess_v2" | |
| }, | |
| "triggerPolicy": { | |
| "condition": "!(inputs.parameter_values['pipelinechannel--use_v2_preprocessing'] == false)" | |
| } | |
| } | |
| } | |
| }, | |
| "inputDefinitions": { | |
| "parameters": { | |
| "pipelinechannel--encryption_spec_key_name": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--evaluation_data_uri": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--export-managed-dataset-export_location": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--model": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--project": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--use_v2_preprocessing": { | |
| "parameterType": "BOOLEAN" | |
| } | |
| } | |
| }, | |
| "outputDefinitions": { | |
| "parameters": { | |
| "pipelinechannel--condition-branches-1-oneof-1": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--condition-branches-1-oneof-2": { | |
| "parameterType": "STRING" | |
| }, | |
| "pipelinechannel--condition-branches-1-oneof-3": { | |
| "parameterType": "STRUCT" | |
| } | |
| } | |
| } | |
| }, | |
| "comp-create-endpoint-and-deploy-model": { | |
| "executorLabel": "exec-create-endpoint-and-deploy-model", | |
| "inputDefinitions": { | |
| "parameters": { | |
| "deploy_model": { | |
| "defaultValue": true, | |
| "description": "Whether to deploy the model to an endpoint. Default\nis ``True``. If ``False``, the model will not be deployed and output\nartifacts will contain empty strings.", | |
| "isOptional": true, | |
| "parameterType": "BOOLEAN" | |
| }, | |
| "display_name": { | |
| "description": "Name of the model (shown in Model Registry).", | |
| "parameterType": "STRING" | |
| }, | |
| "encryption_spec_key_name": { | |
| "defaultValue": "", | |
| "description": "Customer-managed encryption key.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "location": { | |
| "description": "Location for model upload and deployment.", | |
| "parameterType": "STRING" | |
| }, | |
| "model_resource_name": { | |
| "description": "Path to the created Model on Model Registry.", | |
| "parameterType": "STRING" | |
| }, | |
| "project": { | |
| "description": "Name of the GCP project.", | |
| "parameterType": "STRING" | |
| }, | |
| "regional_endpoint": { | |
| "description": "Regional API endpoint.", | |
| "parameterType": "STRING" | |
| }, | |
| "service_account": { | |
| "defaultValue": "", | |
| "description": "If set, then a custom service account will be used.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| } | |
| } | |
| }, | |
| "outputDefinitions": { | |
| "parameters": { | |
| "create_endpoint_gcp_resources": { | |
| "description": "Serialized JSON of GCP resources for\ncreating an endpoint.", | |
| "parameterType": "STRING" | |
| }, | |
| "deploy_model_gcp_resources": { | |
| "description": "Serialized JSON of GCP resources for deploying\nthe model.", | |
| "parameterType": "STRING" | |
| }, | |
| "endpoint_resource_name": { | |
| "description": "Path to the created endpoint on Online Prediction.", | |
| "parameterType": "STRING" | |
| } | |
| } | |
| } | |
| }, | |
| "comp-dataset-encoder": { | |
| "executorLabel": "exec-dataset-encoder", | |
| "inputDefinitions": { | |
| "parameters": { | |
| "dataset_uri": { | |
| "description": "The original dataset needs to be encoded.", | |
| "parameterType": "STRING" | |
| }, | |
| "default_context": { | |
| "defaultValue": "", | |
| "description": "Optional context will be applied to each training\nexample.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "encryption_spec_key_name": { | |
| "defaultValue": "", | |
| "description": "Customer-managed encryption key. If this is set,\nthen all resources created by the CustomJob will be encrypted with the\nprovided encryption key. Note that this is not supported for TPU at\nthe moment.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "image_uri": { | |
| "defaultValue": "us-docker.pkg.dev/vertex-ai-restricted/llm/cloud-llm_dataset_encoding:latest", | |
| "description": "Optional uri of the dataset encoding image.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "large_model_reference": { | |
| "defaultValue": "text-bison@001", | |
| "description": "Large model reference ID indicating which model to\nuse.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "location": { | |
| "defaultValue": "{{$.pipeline_google_cloud_location}}", | |
| "description": "Location to run the job in. Defaults to pipeline region.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "machine_type": { | |
| "defaultValue": "e2-standard-4", | |
| "description": "The type of the machine to provision for the custom job.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "max_examples": { | |
| "defaultValue": 0, | |
| "description": "The maximum number of examples for training or evaluation\ndataset. If 0, no upper limit.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_INTEGER" | |
| }, | |
| "min_examples": { | |
| "defaultValue": 10, | |
| "description": "The minimum number of examples required for training or\nevaluation.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_INTEGER" | |
| }, | |
| "project": { | |
| "description": "Project to run the job in.", | |
| "parameterType": "STRING" | |
| } | |
| } | |
| }, | |
| "outputDefinitions": { | |
| "artifacts": { | |
| "encoded_data": { | |
| "artifactType": { | |
| "schemaTitle": "system.Artifact", | |
| "schemaVersion": "0.0.1" | |
| }, | |
| "description": "Artifact that stores the encoded dataset." | |
| } | |
| }, | |
| "parameters": { | |
| "dataset_path": { | |
| "description": "Path to the training/eval dataset.", | |
| "parameterType": "STRING" | |
| }, | |
| "encoded_dataset_path": { | |
| "description": "Path to the encoded dataset.", | |
| "parameterType": "STRING" | |
| }, | |
| "encoded_dataset_stats": { | |
| "description": "Dictionary of encoded data statistics.", | |
| "parameterType": "STRUCT" | |
| }, | |
| "gcp_resources": { | |
| "description": "Tracker for GCP resources created by this component.", | |
| "parameterType": "STRING" | |
| } | |
| } | |
| } | |
| }, | |
| "comp-dataset-encoder-2": { | |
| "executorLabel": "exec-dataset-encoder-2", | |
| "inputDefinitions": { | |
| "parameters": { | |
| "dataset_uri": { | |
| "description": "The original dataset needs to be encoded.", | |
| "parameterType": "STRING" | |
| }, | |
| "default_context": { | |
| "defaultValue": "", | |
| "description": "Optional context will be applied to each training\nexample.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "encryption_spec_key_name": { | |
| "defaultValue": "", | |
| "description": "Customer-managed encryption key. If this is set,\nthen all resources created by the CustomJob will be encrypted with the\nprovided encryption key. Note that this is not supported for TPU at\nthe moment.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "image_uri": { | |
| "defaultValue": "us-docker.pkg.dev/vertex-ai-restricted/llm/cloud-llm_dataset_encoding:latest", | |
| "description": "Optional uri of the dataset encoding image.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "large_model_reference": { | |
| "defaultValue": "text-bison@001", | |
| "description": "Large model reference ID indicating which model to\nuse.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "location": { | |
| "defaultValue": "{{$.pipeline_google_cloud_location}}", | |
| "description": "Location to run the job in. Defaults to pipeline region.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "machine_type": { | |
| "defaultValue": "e2-standard-4", | |
| "description": "The type of the machine to provision for the custom job.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "max_examples": { | |
| "defaultValue": 0, | |
| "description": "The maximum number of examples for training or evaluation\ndataset. If 0, no upper limit.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_INTEGER" | |
| }, | |
| "min_examples": { | |
| "defaultValue": 10, | |
| "description": "The minimum number of examples required for training or\nevaluation.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_INTEGER" | |
| }, | |
| "project": { | |
| "description": "Project to run the job in.", | |
| "parameterType": "STRING" | |
| } | |
| } | |
| }, | |
| "outputDefinitions": { | |
| "artifacts": { | |
| "encoded_data": { | |
| "artifactType": { | |
| "schemaTitle": "system.Artifact", | |
| "schemaVersion": "0.0.1" | |
| }, | |
| "description": "Artifact that stores the encoded dataset." | |
| } | |
| }, | |
| "parameters": { | |
| "dataset_path": { | |
| "description": "Path to the training/eval dataset.", | |
| "parameterType": "STRING" | |
| }, | |
| "encoded_dataset_path": { | |
| "description": "Path to the encoded dataset.", | |
| "parameterType": "STRING" | |
| }, | |
| "encoded_dataset_stats": { | |
| "description": "Dictionary of encoded data statistics.", | |
| "parameterType": "STRUCT" | |
| }, | |
| "gcp_resources": { | |
| "description": "Tracker for GCP resources created by this component.", | |
| "parameterType": "STRING" | |
| } | |
| } | |
| } | |
| }, | |
| "comp-dataset-encoder-3": { | |
| "executorLabel": "exec-dataset-encoder-3", | |
| "inputDefinitions": { | |
| "parameters": { | |
| "dataset_uri": { | |
| "description": "The original dataset needs to be encoded.", | |
| "parameterType": "STRING" | |
| }, | |
| "default_context": { | |
| "defaultValue": "", | |
| "description": "Optional context will be applied to each training\nexample.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "encryption_spec_key_name": { | |
| "defaultValue": "", | |
| "description": "Customer-managed encryption key. If this is set,\nthen all resources created by the CustomJob will be encrypted with the\nprovided encryption key. Note that this is not supported for TPU at\nthe moment.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "image_uri": { | |
| "defaultValue": "us-docker.pkg.dev/vertex-ai-restricted/llm/cloud-llm_dataset_encoding:latest", | |
| "description": "Optional uri of the dataset encoding image.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "large_model_reference": { | |
| "defaultValue": "text-bison@001", | |
| "description": "Large model reference ID indicating which model to\nuse.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "location": { | |
| "defaultValue": "{{$.pipeline_google_cloud_location}}", | |
| "description": "Location to run the job in. Defaults to pipeline region.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "machine_type": { | |
| "defaultValue": "e2-standard-4", | |
| "description": "The type of the machine to provision for the custom job.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "max_examples": { | |
| "defaultValue": 0, | |
| "description": "The maximum number of examples for training or evaluation\ndataset. If 0, no upper limit.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_INTEGER" | |
| }, | |
| "min_examples": { | |
| "defaultValue": 10, | |
| "description": "The minimum number of examples required for training or\nevaluation.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_INTEGER" | |
| }, | |
| "project": { | |
| "description": "Project to run the job in.", | |
| "parameterType": "STRING" | |
| } | |
| } | |
| }, | |
| "outputDefinitions": { | |
| "artifacts": { | |
| "encoded_data": { | |
| "artifactType": { | |
| "schemaTitle": "system.Artifact", | |
| "schemaVersion": "0.0.1" | |
| }, | |
| "description": "Artifact that stores the encoded dataset." | |
| } | |
| }, | |
| "parameters": { | |
| "dataset_path": { | |
| "description": "Path to the training/eval dataset.", | |
| "parameterType": "STRING" | |
| }, | |
| "encoded_dataset_path": { | |
| "description": "Path to the encoded dataset.", | |
| "parameterType": "STRING" | |
| }, | |
| "encoded_dataset_stats": { | |
| "description": "Dictionary of encoded data statistics.", | |
| "parameterType": "STRUCT" | |
| }, | |
| "gcp_resources": { | |
| "description": "Tracker for GCP resources created by this component.", | |
| "parameterType": "STRING" | |
| } | |
| } | |
| } | |
| }, | |
| "comp-dataset-encoder-4": { | |
| "executorLabel": "exec-dataset-encoder-4", | |
| "inputDefinitions": { | |
| "parameters": { | |
| "dataset_uri": { | |
| "description": "The original dataset needs to be encoded.", | |
| "parameterType": "STRING" | |
| }, | |
| "default_context": { | |
| "defaultValue": "", | |
| "description": "Optional context will be applied to each training\nexample.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "encryption_spec_key_name": { | |
| "defaultValue": "", | |
| "description": "Customer-managed encryption key. If this is set,\nthen all resources created by the CustomJob will be encrypted with the\nprovided encryption key. Note that this is not supported for TPU at\nthe moment.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "image_uri": { | |
| "defaultValue": "us-docker.pkg.dev/vertex-ai-restricted/llm/cloud-llm_dataset_encoding:latest", | |
| "description": "Optional uri of the dataset encoding image.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "large_model_reference": { | |
| "defaultValue": "text-bison@001", | |
| "description": "Large model reference ID indicating which model to\nuse.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "location": { | |
| "defaultValue": "{{$.pipeline_google_cloud_location}}", | |
| "description": "Location to run the job in. Defaults to pipeline region.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "machine_type": { | |
| "defaultValue": "e2-standard-4", | |
| "description": "The type of the machine to provision for the custom job.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "max_examples": { | |
| "defaultValue": 0, | |
| "description": "The maximum number of examples for training or evaluation\ndataset. If 0, no upper limit.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_INTEGER" | |
| }, | |
| "min_examples": { | |
| "defaultValue": 10, | |
| "description": "The minimum number of examples required for training or\nevaluation.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_INTEGER" | |
| }, | |
| "project": { | |
| "description": "Project to run the job in.", | |
| "parameterType": "STRING" | |
| } | |
| } | |
| }, | |
| "outputDefinitions": { | |
| "artifacts": { | |
| "encoded_data": { | |
| "artifactType": { | |
| "schemaTitle": "system.Artifact", | |
| "schemaVersion": "0.0.1" | |
| }, | |
| "description": "Artifact that stores the encoded dataset." | |
| } | |
| }, | |
| "parameters": { | |
| "dataset_path": { | |
| "description": "Path to the training/eval dataset.", | |
| "parameterType": "STRING" | |
| }, | |
| "encoded_dataset_path": { | |
| "description": "Path to the encoded dataset.", | |
| "parameterType": "STRING" | |
| }, | |
| "encoded_dataset_stats": { | |
| "description": "Dictionary of encoded data statistics.", | |
| "parameterType": "STRUCT" | |
| }, | |
| "gcp_resources": { | |
| "description": "Tracker for GCP resources created by this component.", | |
| "parameterType": "STRING" | |
| } | |
| } | |
| } | |
| }, | |
| "comp-deployment-graph": { | |
| "dag": { | |
| "outputs": { | |
| "parameters": { | |
| "endpoint_resource_name": { | |
| "valueFromParameter": { | |
| "outputParameterKey": "endpoint_resource_name", | |
| "producerSubtask": "create-endpoint-and-deploy-model" | |
| } | |
| }, | |
| "model_resource_name": { | |
| "valueFromParameter": { | |
| "outputParameterKey": "model_resource_name", | |
| "producerSubtask": "upload-llm-model" | |
| } | |
| } | |
| } | |
| }, | |
| "tasks": { | |
| "create-endpoint-and-deploy-model": { | |
| "cachingOptions": { | |
| "enableCache": true | |
| }, | |
| "componentRef": { | |
| "name": "comp-create-endpoint-and-deploy-model" | |
| }, | |
| "dependentTasks": [ | |
| "upload-llm-model" | |
| ], | |
| "inputs": { | |
| "parameters": { | |
| "deploy_model": { | |
| "componentInputParameter": "deploy_model" | |
| }, | |
| "display_name": { | |
| "componentInputParameter": "model_display_name" | |
| }, | |
| "encryption_spec_key_name": { | |
| "componentInputParameter": "encryption_spec_key_name" | |
| }, | |
| "location": { | |
| "componentInputParameter": "model_location" | |
| }, | |
| "model_resource_name": { | |
| "taskOutputParameter": { | |
| "outputParameterKey": "model_resource_name", | |
| "producerTask": "upload-llm-model" | |
| } | |
| }, | |
| "project": { | |
| "componentInputParameter": "project" | |
| }, | |
| "regional_endpoint": { | |
| "componentInputParameter": "regional_endpoint" | |
| }, | |
| "service_account": { | |
| "componentInputParameter": "service_account" | |
| } | |
| } | |
| }, | |
| "taskInfo": { | |
| "name": "create-endpoint-and-deploy-model" | |
| } | |
| }, | |
| "upload-llm-model": { | |
| "cachingOptions": { | |
| "enableCache": true | |
| }, | |
| "componentRef": { | |
| "name": "comp-upload-llm-model" | |
| }, | |
| "inputs": { | |
| "artifacts": { | |
| "artifact_uri": { | |
| "componentInputArtifact": "saved_model" | |
| } | |
| }, | |
| "parameters": { | |
| "encryption_spec_key_name": { | |
| "componentInputParameter": "encryption_spec_key_name" | |
| }, | |
| "location": { | |
| "componentInputParameter": "model_location" | |
| }, | |
| "model_display_name": { | |
| "componentInputParameter": "model_display_name" | |
| }, | |
| "model_reference_name": { | |
| "componentInputParameter": "large_model_reference" | |
| }, | |
| "project": { | |
| "componentInputParameter": "project" | |
| }, | |
| "regional_endpoint": { | |
| "componentInputParameter": "regional_endpoint" | |
| }, | |
| "tune_type": { | |
| "componentInputParameter": "tune_type" | |
| } | |
| } | |
| }, | |
| "taskInfo": { | |
| "name": "upload-llm-model" | |
| } | |
| } | |
| } | |
| }, | |
| "inputDefinitions": { | |
| "artifacts": { | |
| "saved_model": { | |
| "artifactType": { | |
| "schemaTitle": "system.Artifact", | |
| "schemaVersion": "0.0.1" | |
| }, | |
| "description": "Output saved_model of adapter tuning." | |
| } | |
| }, | |
| "parameters": { | |
| "deploy_model": { | |
| "defaultValue": true, | |
| "description": "Whether to create the endpoint and deploy the model.", | |
| "isOptional": true, | |
| "parameterType": "BOOLEAN" | |
| }, | |
| "encryption_spec_key_name": { | |
| "defaultValue": "", | |
| "description": "Customer-managed encryption key.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "large_model_reference": { | |
| "defaultValue": "text-bison@001", | |
| "description": "Large model reference ID indicating which model to\nuse.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "model_display_name": { | |
| "description": "Name of the model (shown in model registry).", | |
| "parameterType": "STRING" | |
| }, | |
| "model_location": { | |
| "defaultValue": "us-central1", | |
| "description": "Location for model upload and deployment.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "project": { | |
| "description": "Name of the GCP project.", | |
| "parameterType": "STRING" | |
| }, | |
| "regional_endpoint": { | |
| "defaultValue": "https://us-central1-aiplatform.googleapis.com/ui", | |
| "description": "Regional API endpoint.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "service_account": { | |
| "defaultValue": "", | |
| "description": "If set, then a custom service account will be used.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "tune_type": { | |
| "defaultValue": "", | |
| "description": "Method used to tune the model, e.g. ``distillation``. If\npresent, this value is used to set the ``tune-type`` run label during\nmodel upload.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| } | |
| } | |
| }, | |
| "outputDefinitions": { | |
| "parameters": { | |
| "endpoint_resource_name": { | |
| "parameterType": "STRING" | |
| }, | |
| "model_resource_name": { | |
| "parameterType": "STRING" | |
| } | |
| } | |
| } | |
| }, | |
| "comp-dummy-prompt-validation-component": { | |
| "executorLabel": "exec-dummy-prompt-validation-component", | |
| "inputDefinitions": { | |
| "parameters": { | |
| "encoded_dataset_stats": { | |
| "isOptional": true, | |
| "parameterType": "STRUCT" | |
| } | |
| } | |
| }, | |
| "outputDefinitions": { | |
| "parameters": { | |
| "Output": { | |
| "parameterType": "STRUCT" | |
| } | |
| } | |
| } | |
| }, | |
| "comp-export-managed-dataset": { | |
| "executorLabel": "exec-export-managed-dataset", | |
| "inputDefinitions": { | |
| "parameters": { | |
| "dataset_name": { | |
| "parameterType": "STRING" | |
| }, | |
| "dataset_uri": { | |
| "parameterType": "STRING" | |
| }, | |
| "regional_endpoint": { | |
| "parameterType": "STRING" | |
| } | |
| } | |
| }, | |
| "outputDefinitions": { | |
| "parameters": { | |
| "export_location": { | |
| "parameterType": "STRING" | |
| } | |
| } | |
| } | |
| }, | |
| "comp-large-language-model-tuner": { | |
| "executorLabel": "exec-large-language-model-tuner", | |
| "inputDefinitions": { | |
| "parameters": { | |
| "location": { | |
| "description": "Region in which the launcher for the tuning job is created (not\nthe region for running the whole pipeline).", | |
| "parameterType": "STRING" | |
| }, | |
| "payload_first_part": { | |
| "description": "The first part of the payload to the Custom Job.", | |
| "parameterType": "STRING" | |
| }, | |
| "payload_second_part": { | |
| "description": "The second part of the payload to the Custom Job.", | |
| "parameterType": "STRING" | |
| }, | |
| "project": { | |
| "description": "Name of the project.", | |
| "parameterType": "STRING" | |
| } | |
| } | |
| }, | |
| "outputDefinitions": { | |
| "artifacts": { | |
| "saved_model": { | |
| "artifactType": { | |
| "schemaTitle": "system.Artifact", | |
| "schemaVersion": "0.0.1" | |
| }, | |
| "description": "Artifact for adapter in Private GCS." | |
| }, | |
| "tensorboard_metrics": { | |
| "artifactType": { | |
| "schemaTitle": "system.Artifact", | |
| "schemaVersion": "0.0.1" | |
| }, | |
| "description": "Artifact for saving TensorBoard metrics." | |
| } | |
| }, | |
| "parameters": { | |
| "gcp_resources": { | |
| "description": "Serialized gcp_resources proto.", | |
| "parameterType": "STRING" | |
| } | |
| } | |
| } | |
| }, | |
| "comp-tensorboard-uploader": { | |
| "executorLabel": "exec-tensorboard-uploader", | |
| "inputDefinitions": { | |
| "artifacts": { | |
| "metrics_directory_uri": { | |
| "artifactType": { | |
| "schemaTitle": "system.Artifact", | |
| "schemaVersion": "0.0.1" | |
| } | |
| } | |
| }, | |
| "parameters": { | |
| "experiment_name": { | |
| "parameterType": "STRING" | |
| }, | |
| "location": { | |
| "parameterType": "STRING" | |
| }, | |
| "project": { | |
| "parameterType": "STRING" | |
| }, | |
| "tensorboard_id": { | |
| "parameterType": "STRING" | |
| } | |
| } | |
| }, | |
| "outputDefinitions": { | |
| "parameters": { | |
| "tensorboard_uri": { | |
| "parameterType": "STRING" | |
| } | |
| } | |
| } | |
| }, | |
| "comp-tuning-graph": { | |
| "dag": { | |
| "outputs": { | |
| "artifacts": { | |
| "saved_model": { | |
| "artifactSelectors": [ | |
| { | |
| "outputArtifactKey": "saved_model", | |
| "producerSubtask": "large-language-model-tuner" | |
| } | |
| ] | |
| }, | |
| "tensorboard_metrics": { | |
| "artifactSelectors": [ | |
| { | |
| "outputArtifactKey": "tensorboard_metrics", | |
| "producerSubtask": "large-language-model-tuner" | |
| } | |
| ] | |
| } | |
| } | |
| }, | |
| "tasks": { | |
| "compose-params-for-model-tuner": { | |
| "cachingOptions": { | |
| "enableCache": true | |
| }, | |
| "componentRef": { | |
| "name": "comp-compose-params-for-model-tuner" | |
| }, | |
| "dependentTasks": [ | |
| "condition-branches-1" | |
| ], | |
| "inputs": { | |
| "parameters": { | |
| "display_name": { | |
| "runtimeValue": { | |
| "constant": "large-language-model-tuner-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}" | |
| } | |
| }, | |
| "enable_checkpoint_selection": { | |
| "componentInputParameter": "enable_checkpoint_selection" | |
| }, | |
| "enable_early_stopping": { | |
| "componentInputParameter": "enable_early_stopping" | |
| }, | |
| "encryption_spec_key_name": { | |
| "componentInputParameter": "encryption_spec_key_name" | |
| }, | |
| "evaluation_data_uri": { | |
| "taskOutputParameter": { | |
| "outputParameterKey": "pipelinechannel--condition-branches-1-oneof-2", | |
| "producerTask": "condition-branches-1" | |
| } | |
| }, | |
| "evaluation_interval": { | |
| "componentInputParameter": "evaluation_interval" | |
| }, | |
| "learning_rate": { | |
| "componentInputParameter": "learning_rate" | |
| }, | |
| "learning_rate_multiplier": { | |
| "componentInputParameter": "learning_rate_multiplier" | |
| }, | |
| "location": { | |
| "componentInputParameter": "adapter_training_location" | |
| }, | |
| "max_context_length": { | |
| "componentInputParameter": "max_context_length" | |
| }, | |
| "model": { | |
| "componentInputParameter": "model" | |
| }, | |
| "tpu_training_skip_cmek": { | |
| "componentInputParameter": "tpu_training_skip_cmek" | |
| }, | |
| "train_data_list": { | |
| "componentInputParameter": "train_data_list" | |
| }, | |
| "train_data_uri": { | |
| "taskOutputParameter": { | |
| "outputParameterKey": "pipelinechannel--condition-branches-1-oneof-1", | |
| "producerTask": "condition-branches-1" | |
| } | |
| }, | |
| "train_data_validation_result": { | |
| "taskOutputParameter": { | |
| "outputParameterKey": "pipelinechannel--condition-branches-1-oneof-3", | |
| "producerTask": "condition-branches-1" | |
| } | |
| }, | |
| "train_steps": { | |
| "componentInputParameter": "train_steps" | |
| }, | |
| "tuning_method": { | |
| "componentInputParameter": "tuning_method" | |
| }, | |
| "use_tpu": { | |
| "componentInputParameter": "use_tpu" | |
| } | |
| } | |
| }, | |
| "taskInfo": { | |
| "name": "compose-params-for-model-tuner" | |
| } | |
| }, | |
| "condition-branches-1": { | |
| "componentRef": { | |
| "name": "comp-condition-branches-1" | |
| }, | |
| "dependentTasks": [ | |
| "export-managed-dataset" | |
| ], | |
| "inputs": { | |
| "parameters": { | |
| "pipelinechannel--encryption_spec_key_name": { | |
| "componentInputParameter": "encryption_spec_key_name" | |
| }, | |
| "pipelinechannel--evaluation_data_uri": { | |
| "componentInputParameter": "evaluation_data_uri" | |
| }, | |
| "pipelinechannel--export-managed-dataset-export_location": { | |
| "taskOutputParameter": { | |
| "outputParameterKey": "export_location", | |
| "producerTask": "export-managed-dataset" | |
| } | |
| }, | |
| "pipelinechannel--model": { | |
| "componentInputParameter": "model" | |
| }, | |
| "pipelinechannel--project": { | |
| "componentInputParameter": "project" | |
| }, | |
| "pipelinechannel--use_v2_preprocessing": { | |
| "componentInputParameter": "use_v2_preprocessing" | |
| } | |
| } | |
| }, | |
| "taskInfo": { | |
| "name": "condition-branches-1" | |
| } | |
| }, | |
| "export-managed-dataset": { | |
| "cachingOptions": { | |
| "enableCache": true | |
| }, | |
| "componentRef": { | |
| "name": "comp-export-managed-dataset" | |
| }, | |
| "inputs": { | |
| "parameters": { | |
| "dataset_name": { | |
| "componentInputParameter": "dataset_name" | |
| }, | |
| "dataset_uri": { | |
| "componentInputParameter": "dataset_uri" | |
| }, | |
| "regional_endpoint": { | |
| "componentInputParameter": "regional_endpoint" | |
| } | |
| } | |
| }, | |
| "taskInfo": { | |
| "name": "export-managed-dataset" | |
| } | |
| }, | |
| "large-language-model-tuner": { | |
| "cachingOptions": { | |
| "enableCache": true | |
| }, | |
| "componentRef": { | |
| "name": "comp-large-language-model-tuner" | |
| }, | |
| "dependentTasks": [ | |
| "compose-params-for-model-tuner" | |
| ], | |
| "inputs": { | |
| "parameters": { | |
| "location": { | |
| "componentInputParameter": "adapter_training_location" | |
| }, | |
| "payload_first_part": { | |
| "taskOutputParameter": { | |
| "outputParameterKey": "payload_first_part", | |
| "producerTask": "compose-params-for-model-tuner" | |
| } | |
| }, | |
| "payload_second_part": { | |
| "taskOutputParameter": { | |
| "outputParameterKey": "payload_second_part", | |
| "producerTask": "compose-params-for-model-tuner" | |
| } | |
| }, | |
| "project": { | |
| "componentInputParameter": "project" | |
| } | |
| } | |
| }, | |
| "taskInfo": { | |
| "name": "large-language-model-tuner" | |
| } | |
| } | |
| } | |
| }, | |
| "inputDefinitions": { | |
| "parameters": { | |
| "adapter_training_location": { | |
| "defaultValue": "europe-west4", | |
| "description": "Location adapter training.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "dataset_name": { | |
| "defaultValue": "", | |
| "description": "Name of training dataset if importing from managed dataset.\nOnly this or `dataset_uri` should be set.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "dataset_uri": { | |
| "defaultValue": "", | |
| "description": "GCS URI of training dataset. Only this or `dataset_name`\nshould be set.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "enable_checkpoint_selection": { | |
| "defaultValue": "default", | |
| "description": "One of 'true', 'false', or 'default'. If set\nto 'true' and if evaluation_data_uri is set, select the best tuned\ncheckpoint. If 'default' is set, checkpoint selection is enabled for\nbison@001 models, and disabled for others.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "enable_early_stopping": { | |
| "defaultValue": true, | |
| "description": "If set to True, and if evaluation_data_uri is set,\nthen early stopping will be enabled. If early stopping is enabled, the\nmax recommended train_steps is 1,000.", | |
| "isOptional": true, | |
| "parameterType": "BOOLEAN" | |
| }, | |
| "encryption_spec_key_name": { | |
| "defaultValue": "", | |
| "description": "Customer-managed encryption key.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "evaluation_data_uri": { | |
| "defaultValue": "", | |
| "description": "GCS URI of eval dataset.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "evaluation_interval": { | |
| "defaultValue": 20, | |
| "description": "Evaluation after every this number of tuning steps.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_INTEGER" | |
| }, | |
| "learning_rate": { | |
| "defaultValue": -1, | |
| "description": "Learning rate hyperparameter for tuning. If this value is\nnegative, then the default learning rate per model per tuning_method\nwill be used. This parameter will be deprecated.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_DOUBLE" | |
| }, | |
| "learning_rate_multiplier": { | |
| "defaultValue": 1, | |
| "description": "Learning rate multiplier for tuning. Under the\nhood, the actual \"learning rate\" will be learning_rate_multiplier * the\n\"recommended learning rate\" per model, so customers don't have to keep\ntrack of the recommended learning rate for each model or tuning method.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_DOUBLE" | |
| }, | |
| "max_context_length": { | |
| "defaultValue": "", | |
| "description": "The max context length used for tuning. Can be either\n'8k' or '32k'.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "model": { | |
| "defaultValue": "text-bison@001", | |
| "description": "Model name indicating which model to tuning.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "project": { | |
| "description": "Name of the GCP project.", | |
| "parameterType": "STRING" | |
| }, | |
| "regional_endpoint": { | |
| "defaultValue": "https://us-central1-aiplatform.googleapis.com/ui", | |
| "description": "Regional API endpoint.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "tpu_training_skip_cmek": { | |
| "defaultValue": false, | |
| "description": "If set to True, CMEK will be ignored during TPU\ntraining step.", | |
| "isOptional": true, | |
| "parameterType": "BOOLEAN" | |
| }, | |
| "train_data_list": { | |
| "description": "List of dict representation of training_data\nwith its key restricted to `path` and `weight`., e.g., `[{\"path\":\n\"path1\", \"weight\": 0.3}, {\"path\": \"path2\", \"weight\": 0.3}]`. For now,\neach path should be tfrecord converted from previous components.", | |
| "isOptional": true, | |
| "parameterType": "LIST" | |
| }, | |
| "train_steps": { | |
| "defaultValue": 300, | |
| "description": "Number of training steps for tuning step.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_INTEGER" | |
| }, | |
| "tuning_method": { | |
| "defaultValue": "tune_v2", | |
| "description": "The adapter tuning method. Available methods: tune,\ntune_v2.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "use_tpu": { | |
| "defaultValue": true, | |
| "description": "If set, adapter tuning will be running on TPU.", | |
| "isOptional": true, | |
| "parameterType": "BOOLEAN" | |
| }, | |
| "use_v2_preprocessing": { | |
| "defaultValue": false, | |
| "description": "Whether to run v2 preprocessing steps.", | |
| "isOptional": true, | |
| "parameterType": "BOOLEAN" | |
| } | |
| } | |
| }, | |
| "outputDefinitions": { | |
| "artifacts": { | |
| "saved_model": { | |
| "artifactType": { | |
| "schemaTitle": "system.Artifact", | |
| "schemaVersion": "0.0.1" | |
| } | |
| }, | |
| "tensorboard_metrics": { | |
| "artifactType": { | |
| "schemaTitle": "system.Artifact", | |
| "schemaVersion": "0.0.1" | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "comp-upload-llm-model": { | |
| "executorLabel": "exec-upload-llm-model", | |
| "inputDefinitions": { | |
| "artifacts": { | |
| "artifact_uri": { | |
| "artifactType": { | |
| "schemaTitle": "system.Artifact", | |
| "schemaVersion": "0.0.1" | |
| }, | |
| "description": "KFP Artifact for adapter." | |
| } | |
| }, | |
| "parameters": { | |
| "encryption_spec_key_name": { | |
| "defaultValue": "", | |
| "description": "Customer-managed encryption key.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "location": { | |
| "description": "Location for model upload and deployment.", | |
| "parameterType": "STRING" | |
| }, | |
| "model_display_name": { | |
| "description": "Name of the model (shown in Model Registry).", | |
| "parameterType": "STRING" | |
| }, | |
| "model_reference_name": { | |
| "description": "Large model reference name.", | |
| "parameterType": "STRING" | |
| }, | |
| "project": { | |
| "description": "Name of the GCP project.", | |
| "parameterType": "STRING" | |
| }, | |
| "regional_endpoint": { | |
| "description": "Regional API endpoint.", | |
| "parameterType": "STRING" | |
| }, | |
| "tune_type": { | |
| "defaultValue": "", | |
| "description": "Method used to tune the model, e.g. ``distillation``. If\npresent, this value is used to set the ``tune-type`` run label during\nmodel upload.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "upload_model": { | |
| "defaultValue": true, | |
| "description": "Whether to upload the model to the Model Registry. Default\nis ``True``. If ``False``, the model will not be uploaded and output\nartifacts will contain empty strings.", | |
| "isOptional": true, | |
| "parameterType": "BOOLEAN" | |
| } | |
| } | |
| }, | |
| "outputDefinitions": { | |
| "parameters": { | |
| "gcp_resources": { | |
| "description": "Serialized JSON of `gcp_resources`.", | |
| "parameterType": "STRING" | |
| }, | |
| "model_resource_name": { | |
| "description": "Path to the created Model on Model Registry.", | |
| "parameterType": "STRING" | |
| } | |
| } | |
| } | |
| }, | |
| "comp-validate-pipeline": { | |
| "executorLabel": "exec-validate-pipeline", | |
| "inputDefinitions": { | |
| "parameters": { | |
| "accelerator_type": { | |
| "defaultValue": "", | |
| "description": "One of 'TPU' or 'GPU'. If 'TPU' is specified, tuning\nruns in europe-west4, else in us-central1.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "api_endpoint": { | |
| "defaultValue": "aiplatform.googleapis.com/ui", | |
| "description": "API endpoint.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "enable_tpu_v5e": { | |
| "defaultValue": false, | |
| "description": "Whether to enable TPU V5e tuning or not.", | |
| "isOptional": true, | |
| "parameterType": "BOOLEAN" | |
| }, | |
| "encryption_spec_key_name": { | |
| "defaultValue": "", | |
| "description": "If set, CMEK will be used.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "evaluation_data_uri": { | |
| "defaultValue": "", | |
| "description": "GCS URI of eval dataset. If evaluation dataset is\nprovided, the best tuned checkpoint will be selected.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "evaluation_output_root_dir": { | |
| "defaultValue": "", | |
| "description": "Root GCS URI of evaluation outputs.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "location": { | |
| "defaultValue": "{{$.pipeline_google_cloud_location}}", | |
| "description": "Region in which all the components except for tuning job should\nrun.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "max_context_length": { | |
| "defaultValue": "", | |
| "description": "The max context length used for tuning. Can be either\n'8k' or '32k'.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "model": { | |
| "defaultValue": "text-bison-001", | |
| "description": "Large language reference model to use (default: `text-bison-001`).", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "pipeline_region": { | |
| "defaultValue": "{{$.pipeline_google_cloud_location}}", | |
| "description": "The region the pipeline runs in.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "tensorboard_resource_id": { | |
| "defaultValue": "", | |
| "description": "The Vertex AI TensorBoard Resource/Instance ID.\nIf this parameter is set, then a Vertex AI TensorBoard Experiment will\nbe launched.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "tpu_training_skip_cmek": { | |
| "defaultValue": false, | |
| "description": "If True, skip CMEK setup for TPU training.", | |
| "isOptional": true, | |
| "parameterType": "BOOLEAN" | |
| }, | |
| "tuning_method": { | |
| "defaultValue": "tune", | |
| "description": "Tuning method.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| } | |
| } | |
| }, | |
| "outputDefinitions": { | |
| "parameters": { | |
| "adapter_training_location": { | |
| "parameterType": "STRING" | |
| }, | |
| "model_at_version": { | |
| "parameterType": "STRING" | |
| }, | |
| "model_dash_version": { | |
| "parameterType": "STRING" | |
| }, | |
| "model_location": { | |
| "parameterType": "STRING" | |
| }, | |
| "regional_endpoint": { | |
| "parameterType": "STRING" | |
| }, | |
| "service_account": { | |
| "parameterType": "STRING" | |
| }, | |
| "use_tpu": { | |
| "parameterType": "BOOLEAN" | |
| }, | |
| "use_v2_preprocessing": { | |
| "parameterType": "BOOLEAN" | |
| }, | |
| "with_batch_prediction_and_evaluation": { | |
| "parameterType": "BOOLEAN" | |
| }, | |
| "with_tensorboard": { | |
| "parameterType": "BOOLEAN" | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "deploymentSpec": { | |
| "executors": { | |
| "exec-compose-params-for-model-tuner": { | |
| "container": { | |
| "args": [ | |
| "--executor_input", | |
| "{{$}}", | |
| "--function_to_execute", | |
| "compose_params_for_model_tuner" | |
| ], | |
| "command": [ | |
| "sh", | |
| "-ec", | |
| "program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", | |
| "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef compose_params_for_model_tuner(\n display_name: str,\n train_data_uri: str,\n tuning_method: str = 'tune_v2',\n location: str = 'us-central1',\n model: str = 'text-bison@001',\n train_steps: int = 0,\n adapter_length: int = 0,\n learning_rate: float = -1,\n learning_rate_multiplier: float = 1.0,\n evaluation_data_uri: str = '',\n evaluation_interval: int = 20,\n enable_early_stopping: bool = True,\n use_tpu: bool = True,\n encryption_spec_key_name: str = '',\n tpu_training_skip_cmek: bool = False,\n pipeline_region: str = '{{$.pipeline_google_cloud_location}}',\n enable_checkpoint_selection: str = 'default',\n train_data_validation_result: Optional[dict] = None,\n train_data_list: Optional[list] = None,\n max_context_length: str = '',\n) -> NamedTuple('outputs',\n payload_first_part=str,\n payload_second_part=str,\n location=str):\n \"\"\"Composes the parameters for running model tuner.\n\n Args:\n display_name: The display name of the model tuner.\n tuning_method: The model tuning method. Available methods: tune,\n tune_v2.\n train_data_uri: File or file pattern containing train data.\n location: Region in which the launcher for the tuning job is created (not\n the region for running the whole pipeline).\n model: Model type to use.\n train_steps: Number of training steps.\n adapter_length: Adapter length.\n learning_rate: Learning rate hyperparameter. When learning rate is less\n than 0, it uses the default learning rate per tuning method per model.\n This parameter will be deprecated.\n learning_rate_multiplier: Learning rate multiplier for tuning. Under the\n hood, the actual \"learning rate\" will be learning_rate_multiplier *\n the \"recommended learning rate\" per model, so customers don't have to\n keep track of the recommended learning rate for each model or tuning\n method.\n evaluation_data_uri: GCS URI of eval dataset.\n evaluation_interval: Evaluation after every this number of tuning steps.\n enable_early_stopping: If set to True, and if evaluation_data_uri is set,\n then early stopping will be enabled. If early stopping is enabled, the\n max recommended train_steps is 1,000.\n use_tpu: If True, use TPU. Otherwise use GPU.\n encryption_spec_key_name: Customer-managed encryption key. If this is set,\n then all resources created by the CustomJob will be encrypted with the\n provided encryption key. Note that this is not supported for TPU at\n the moment.\n tpu_training_skip_cmek: If True, skip CMEK setup for TPU training.\n pipeline_region: The region the pipeline runs in.\n enable_checkpoint_selection: One of 'true', 'false', or 'default'. If set\n to 'true' and if evaluation_data_uri is set, select the best tuned\n checkpoint. If 'default' is set, checkpoint selection is enabled\n for bison@001 models, and disabled for others.\n train_data_validation_result: Json object containing validation result of\n the training dataset.\n train_data_list: List of dict representation of training_data\n with its key restricted to `path` and `weight`.\n max_context_length: The max context length used for tuning. Can be either\n '8k' or '32k'.\n\n Returns:\n outputs: Structured parameters for creating a Custom Job to run the\n adapter training job.\n \"\"\"\n import json\n import logging\n from typing import Any\n import dataclasses\n import sys\n\n def return_fixed_inputs_outputs_lengths(outputs_length: int, is_32k: bool):\n if is_32k:\n combinations = [(30720, 2048), (28672, 4096), (24576, 8192)]\n else:\n combinations = [(7168, 1024), (6144, 2048)]\n\n for c in combinations:\n if c[1] >= outputs_length:\n return c\n\n return combinations[-1]\n\n latest_models = frozenset(['gemini-pro',])\n v3_models = frozenset([\n 'text-bison@private',\n 'chat-bison@private',\n 'code-bison@private',\n 'text-bison@002',\n 'chat-bison@002',\n 'code-bison@002',\n 'codechat-bison@002',\n 'text-bison-32k@002',\n 'chat-bison-32k@002',\n 'code-bison-32k@002',\n 'codechat-bison-32k@002',\n 'MedLM',\n ])\n\n location_to_accelerator = {\n 'us-west1': 'tpu-v5e-pod',\n 'europe-west4': 'tpu-v3-pod',\n 'us-central1': 'gpu',\n }\n\n @dataclasses.dataclass(frozen=True, eq=True)\n class TunedModel:\n model: str\n accelerator: str = 'tpu-v3-pod'\n tuning_method: str = 'tune_v2'\n\n @dataclasses.dataclass(frozen=True)\n class TuneParams:\n train_steps: int = 300\n learning_rate: float = -1\n adapter_length: int = 1\n inputs_length: int = 8192\n outputs_length: int = 1024\n per_core_batch_size: Any = None\n ici_mesh_shape: Any = None\n enable_checkpoint_selection: bool = False\n\n class GpuTuneParams(TuneParams):\n\n def __init__(self, per_core_batch_size=None, ici_mesh_shape=None, **kwargs):\n super().__init__(\n per_core_batch_size=per_core_batch_size or 1,\n ici_mesh_shape=ici_mesh_shape or [1, 1, 8],\n **kwargs,\n )\n\n class TpuTuneParams(TuneParams):\n\n def __init__(\n self,\n per_core_batch_size=None,\n ici_mesh_shape=None,\n accelerator='tpu-v3-pod',\n **kwargs,\n ):\n default_ici_mesh_shape = (\n [1, 4, 16] if accelerator == 'tpu-v3-pod' else [1, 8, 8]\n )\n super().__init__(\n per_core_batch_size=per_core_batch_size or 0.375,\n ici_mesh_shape=ici_mesh_shape or default_ici_mesh_shape,\n **kwargs,\n )\n legacy_tune_params_tpu = TpuTuneParams(\n train_steps=1000,\n adapter_length=20,\n inputs_length=4096,\n outputs_length=1024,\n per_core_batch_size=0.125,\n ici_mesh_shape=[1, 64, 1],\n learning_rate=3.0,\n )\n legacy_tune_params_gpu = GpuTuneParams(\n train_steps=1000,\n adapter_length=20,\n inputs_length=2048,\n outputs_length=512,\n learning_rate=3.0,\n )\n tune_params_private_tpu = TpuTuneParams(\n per_core_batch_size=0.375 if location_to_accelerator[location] == 'tpu-v3-pod' else 0.125,\n inputs_length=8192 - 1024,\n outputs_length=1024,\n accelerator=location_to_accelerator[location]\n )\n tune_params_private_gpu = GpuTuneParams(inputs_length=8192 - 1024,\n outputs_length=1024)\n\n tune_params_gemini_pro_tpu = tune_params_private_tpu\n tune_params_gemini_pro_gpu = tune_params_private_gpu\n max_context = 8 * 1024\n use_32k = (\n model == 'gemini-pro' and max_context_length.lower() == '32k'\n ) or model.endswith('32k@002')\n if use_32k:\n max_context = 32 * 1024\n tune_params_gemini_pro_tpu = TpuTuneParams(\n per_core_batch_size=0.03125,\n inputs_length=28*1024,\n outputs_length=4*1024,\n ici_mesh_shape=[1, 8, 32],\n accelerator=location_to_accelerator[location],\n )\n tune_params_gemini_pro_gpu = GpuTuneParams(per_core_batch_size=0.25,\n inputs_length=28*1024,\n outputs_length=4*1024)\n default_per_model_params = {\n TunedModel(\n tuning_method='tune',\n model='text-bison-001',\n ): legacy_tune_params_tpu,\n TunedModel(\n tuning_method='tune',\n model='text-bison-001',\n accelerator='gpu',\n ): legacy_tune_params_gpu,\n TunedModel(\n tuning_method='tune',\n model='text-bison@001',\n ): legacy_tune_params_tpu,\n TunedModel(\n tuning_method='tune',\n model='text-bison@001',\n accelerator='gpu',\n ): legacy_tune_params_gpu,\n TunedModel(\n model='text-bison@001',\n ): TpuTuneParams(enable_checkpoint_selection=True),\n TunedModel(\n model='text-bison@001',\n accelerator='tpu-v5e-pod',\n ): TpuTuneParams(enable_checkpoint_selection=True,\n accelerator='tpu-v5e-pod',\n per_core_batch_size=0.125,),\n TunedModel(\n model='text-bison@001',\n accelerator='gpu',\n ): GpuTuneParams(enable_checkpoint_selection=True),\n TunedModel(\n model='code-bison@001',\n ): TpuTuneParams(\n inputs_length=6144,\n outputs_length=2048,\n enable_checkpoint_selection=True,\n ),\n TunedModel(\n model='code-bison@001',\n accelerator='tpu-v5e-pod',\n ): TpuTuneParams(\n inputs_length=6144,\n outputs_length=2048,\n accelerator='tpu-v5e-pod',\n enable_checkpoint_selection=True,\n per_core_batch_size=0.125,\n ),\n TunedModel(\n model='code-bison@001',\n accelerator='gpu',\n ): GpuTuneParams(\n inputs_length=6144,\n outputs_length=2048,\n enable_checkpoint_selection=True,\n ),\n TunedModel(\n model='chat-bison@001',\n ): TpuTuneParams(enable_checkpoint_selection=True),\n TunedModel(\n model='chat-bison@001',\n accelerator='tpu-v5e-pod',\n ): TpuTuneParams(enable_checkpoint_selection=True,\n accelerator='tpu-v5e-pod',\n per_core_batch_size=0.125,),\n TunedModel(\n model='chat-bison@001',\n accelerator='gpu',\n ): GpuTuneParams(enable_checkpoint_selection=True),\n TunedModel(\n model='codechat-bison@001',\n ): TpuTuneParams(\n inputs_length=6144,\n outputs_length=2048,\n enable_checkpoint_selection=True,\n ),\n TunedModel(\n model='codechat-bison@001',\n accelerator='tpu-v5e-pod',\n ): TpuTuneParams(\n inputs_length=6144,\n outputs_length=2048,\n enable_checkpoint_selection=True,\n accelerator='tpu-v5e-pod',\n per_core_batch_size=0.125,\n ),\n TunedModel(\n model='codechat-bison@001',\n accelerator='gpu',\n ): GpuTuneParams(\n inputs_length=6144,\n outputs_length=2048,\n enable_checkpoint_selection=True,\n ),\n TunedModel(\n model='text-bison',\n ): TpuTuneParams(per_core_batch_size=0.125),\n TunedModel(\n model='text-bison',\n accelerator='tpu-v5e-pod',\n ): TpuTuneParams(per_core_batch_size=0.125,\n accelerator='tpu-v5e-pod',),\n TunedModel(\n model='text-bison',\n accelerator='gpu',\n ): GpuTuneParams(per_core_batch_size=0.375),\n TunedModel(\n model='chat-bison',\n ): TpuTuneParams(per_core_batch_size=0.125),\n TunedModel(\n model='chat-bison',\n accelerator='tpu-v5e-pod',\n ): TpuTuneParams(per_core_batch_size=0.125,\n accelerator='tpu-v5e-pod',),\n TunedModel(\n model='chat-bison',\n accelerator='gpu',\n ): GpuTuneParams(per_core_batch_size=0.375),\n TunedModel(\n model='code-bison',\n ): TpuTuneParams(per_core_batch_size=0.125, inputs_length=6144,\n outputs_length=2048),\n TunedModel(\n model='code-bison',\n accelerator='tpu-v5e-pod',\n ): TpuTuneParams(per_core_batch_size=0.125, inputs_length=6144,\n outputs_length=2048, accelerator='tpu-v5e-pod',),\n TunedModel(\n model='code-bison',\n accelerator='gpu',\n ): GpuTuneParams(per_core_batch_size=0.375, inputs_length=6144,\n outputs_length=2048),\n TunedModel(\n model='codechat-bison',\n ): TpuTuneParams(per_core_batch_size=0.125, inputs_length=6144,\n outputs_length=2048),\n TunedModel(\n model='codechat-bison',\n accelerator='tpu-v5e-pod',\n ): TpuTuneParams(per_core_batch_size=0.125, inputs_length=6144,\n outputs_length=2048, accelerator='tpu-v5e-pod',),\n TunedModel(\n model='codechat-bison',\n accelerator='gpu',\n ): GpuTuneParams(per_core_batch_size=0.375, inputs_length=6144,\n outputs_length=2048),\n TunedModel(\n model='text-bison@private',\n ): tune_params_private_tpu,\n TunedModel(\n model='text-bison@private',\n accelerator='tpu-v5e-pod',\n ): tune_params_private_tpu,\n TunedModel(\n model='text-bison@private',\n accelerator='gpu',\n ): tune_params_private_gpu,\n TunedModel(\n model='chat-bison@private',\n ): tune_params_private_tpu,\n TunedModel(\n model='chat-bison@private',\n accelerator='tpu-v5e-pod',\n ): tune_params_private_tpu,\n TunedModel(\n model='chat-bison@private',\n accelerator='gpu',\n ): tune_params_private_gpu,\n TunedModel(\n model='code-bison@private',\n ): tune_params_private_tpu,\n TunedModel(\n model='code-bison@private',\n accelerator='tpu-v5e-pod',\n ): tune_params_private_tpu,\n TunedModel(\n model='code-bison@private',\n accelerator='gpu',\n ): tune_params_private_gpu,\n TunedModel(\n model='gemini-pro',\n ): tune_params_gemini_pro_tpu,\n TunedModel(\n model='gemini-pro',\n accelerator='tpu-v5e-pod',\n ): tune_params_gemini_pro_tpu,\n TunedModel(\n model='text-bison@002',\n ): tune_params_private_tpu,\n TunedModel(\n model='text-bison@002',\n accelerator='tpu-v5e-pod',\n ): tune_params_private_tpu,\n TunedModel(\n model='text-bison@002',\n accelerator='gpu',\n ): tune_params_private_gpu,\n TunedModel(\n model='chat-bison@002',\n ): tune_params_private_tpu,\n TunedModel(\n model='chat-bison@002',\n accelerator='tpu-v5e-pod',\n ): tune_params_private_tpu,\n TunedModel(\n model='chat-bison@002',\n accelerator='gpu',\n ): tune_params_private_gpu,\n TunedModel(\n model='code-bison@002',\n ): tune_params_private_tpu,\n TunedModel(\n model='code-bison@002',\n accelerator='tpu-v5e-pod',\n ): tune_params_private_tpu,\n TunedModel(\n model='code-bison@002',\n accelerator='gpu',\n ): tune_params_private_gpu,\n TunedModel(\n model='codechat-bison@002',\n ): tune_params_private_tpu,\n TunedModel(\n model='codechat-bison@002',\n accelerator='tpu-v5e-pod',\n ): tune_params_private_tpu,\n TunedModel(\n model='codechat-bison@002',\n accelerator='gpu',\n ): tune_params_private_gpu,\n TunedModel(\n model='text-bison-32k@002',\n ): tune_params_gemini_pro_tpu,\n TunedModel(\n model='text-bison-32k@002',\n accelerator='gpu',\n ): tune_params_gemini_pro_gpu,\n TunedModel(\n model='chat-bison-32k@002',\n ): tune_params_gemini_pro_tpu,\n TunedModel(\n model='chat-bison-32k@002',\n accelerator='gpu',\n ): tune_params_gemini_pro_gpu,\n TunedModel(\n model='code-bison-32k@002',\n ): tune_params_gemini_pro_tpu,\n TunedModel(\n model='code-bison-32k@002',\n accelerator='gpu',\n ): tune_params_gemini_pro_gpu,\n TunedModel(\n model='codechat-bison-32k@002',\n ): tune_params_gemini_pro_tpu,\n TunedModel(\n model='codechat-bison-32k@002',\n accelerator='gpu',\n ): tune_params_gemini_pro_gpu,\n TunedModel(\n model='MedLM',\n ): tune_params_private_tpu,\n TunedModel(\n model='MedLM',\n accelerator='gpu',\n ): tune_params_private_gpu,\n TunedModel(\n model='gemini-pro-vision',\n ): tune_params_private_tpu,\n }\n\n try:\n if tuning_method not in ['tune', 'tune_v2']:\n raise ValueError(f'Unsupported tuning method [{tuning_method}].')\n\n default_params = default_per_model_params.get(\n TunedModel(\n tuning_method=tuning_method,\n model=model,\n accelerator=location_to_accelerator[location],\n ),\n TpuTuneParams() if use_tpu else GpuTuneParams(),\n )\n\n inputs_length = default_params.inputs_length\n outputs_length = default_params.outputs_length\n if '001' not in model and train_data_validation_result:\n max_train_inputs_length = train_data_validation_result.get(\n 'max_user_input_token_length', default_params.inputs_length)\n max_train_outputs_length = train_data_validation_result.get(\n 'max_user_output_token_length', default_params.outputs_length)\n if max_train_inputs_length > 0 and max_train_outputs_length > 0:\n inputs_length = max_train_inputs_length\n outputs_length = max(64, max_train_outputs_length)\n if outputs_length >= max_context:\n inputs_length = default_params.inputs_length\n outputs_length = default_params.outputs_length\n elif inputs_length + outputs_length > max_context:\n inputs_length = max_context - outputs_length\n\n # Round total context length to a multiple of 128 for TPU tuning\n if (inputs_length + outputs_length) % 128 != 0 and use_tpu:\n inputs_length = (\n (inputs_length + outputs_length) // 128 + 1\n ) * 128 - outputs_length\n\n if ('002' in model or model == 'gemini-pro' or model == 'MedLM'\n ) and not use_tpu:\n inputs_length, outputs_length = return_fixed_inputs_outputs_lengths(\n outputs_length, use_32k\n )\n\n executor_line = '--executor_input={{<PLACE_HOLDER>}}'\n\n image_uri_prefix = 'us-docker.pkg.dev/vertex-ai-restricted/llm/llm-pets'\n args = [\n tuning_method,\n f'--model={model}',\n '--task=generation',\n f'--train_data_path={train_data_uri}',\n '--feature_keys=input_text',\n '--label_key=output_text',\n (\n f\"\"\"--train_steps={train_steps if train_steps > 0\n else default_params.train_steps}\"\"\"\n ),\n f'--inputs_length={inputs_length}',\n f'--outputs_length={outputs_length}',\n f'--ici_mesh_shape={default_params.ici_mesh_shape}',\n f'--per_core_batch_size={default_params.per_core_batch_size}',\n (\n f\"\"\"--adapter_length={adapter_length if adapter_length > 0\n else default_params.adapter_length}\"\"\"\n ),\n ]\n\n # If specifies, will override train_data_path\n if train_data_list:\n args.append(\n f'--train_data_json_array={json.dumps(train_data_list, sort_keys=True)}'\n )\n\n if 'tune' == tuning_method:\n if learning_rate >= 0:\n args.append(f'--learning_rate={learning_rate * learning_rate_multiplier}')\n elif learning_rate_multiplier != 1.0:\n args.append(f\"\"\"--learning_rate={\n default_params.learning_rate * learning_rate_multiplier}\"\"\")\n elif 'tune_v2' == tuning_method:\n args.append(f'--train_evaluation_interval={evaluation_interval}')\n if evaluation_data_uri:\n args.append(f'--validation_data_path={evaluation_data_uri}')\n if enable_early_stopping:\n args.append(\n '--early_stopping=metric_name='\n '/eval_fraction_of_correct_next_step_preds,mode=max,patience=8,'\n 'relative_tolerance=0.001'\n )\n checkpoint_selector = '{}'\n if enable_checkpoint_selection.lower() == 'true' or (\n enable_checkpoint_selection.lower() == 'default'\n and default_params.enable_checkpoint_selection\n ):\n checkpoint_selector = json.dumps(\n {\n 'checkpoint_selector_cls': 'best_single_metric',\n 'metric_name': '/eval_fraction_of_correct_next_step_preds',\n 'metric_mode': 'max',\n },\n sort_keys=True,\n )\n args.append(f'--checkpoint_selector={checkpoint_selector}')\n if learning_rate >= 0:\n args.append(f'--learning_rate={learning_rate}')\n if learning_rate_multiplier >= 0:\n args.append(f'--learning_rate_multiplier={learning_rate_multiplier}')\n if use_32k:\n args.append('--max_context_length=32k')\n\n args.append(executor_line)\n\n if use_tpu:\n if location == 'us-west1' and not use_32k and tuning_method == 'tune_v2':\n machine_spec = {'machineType': 'ct5lp-hightpu-4t', 'tpuTopology': '8x8'}\n else:\n machine_spec = {\n 'machine_type': 'cloud-tpu',\n 'accelerator_type': 'TPU_V3',\n 'accelerator_count': 64 if not use_32k else 256,\n }\n disk_spec = {'boot_disk_type': 'pd-ssd', 'boot_disk_size_gb': 100}\n if 'tune' == tuning_method:\n image_uri = f'{image_uri_prefix}-tpu:v1.0.0'\n else: # Default to tune_v2\n image_uri = f'{image_uri_prefix}-tpu:v2.2.0'\n if model in latest_models:\n image_uri = 'us-docker.pkg.dev/vertex-ai-restricted/genai-tuning-restricted/llm-pets-tpu-experiment:v1.0.0'\n elif model in v3_models:\n image_uri = f'{image_uri_prefix}-tpu:v3.0.0'\n else:\n machine_spec = {\n 'machine_type': 'a2-ultragpu-8g',\n 'accelerator_type': 'NVIDIA_A100_80GB',\n 'accelerator_count': 8,\n }\n disk_spec = {'boot_disk_type': 'pd-ssd', 'boot_disk_size_gb': 500}\n if 'tune' == tuning_method:\n image_uri = f'{image_uri_prefix}-gpu:v1.0.0'\n else: # Default to tune_v2\n image_uri = f'{image_uri_prefix}-gpu:v2.2.0'\n if model in v3_models:\n image_uri = f'{image_uri_prefix}-gpu:v3.0.0'\n custom_job_payload = {\n 'display_name': display_name,\n 'labels': {'vertex-internal-enable-custom-job-retries': ''},\n 'job_spec': {\n 'worker_pool_specs': [{\n 'replica_count': 1,\n 'machine_spec': machine_spec,\n 'disk_spec': disk_spec,\n 'container_spec': {'image_uri': image_uri, 'args': args},\n }],\n 'scheduling': {\n 'disable_retries': False,\n },\n },\n }\n\n if tuning_method == 'tune_v2':\n custom_job_payload['job_spec'][\n 'protected_artifact_location_id'\n ] = pipeline_region\n\n # Set up CMEK\n if encryption_spec_key_name:\n if use_tpu:\n if tpu_training_skip_cmek:\n logging.warning(\n 'CMEK will not be set for TPU training. If this is not expected, '\n 'please run with GPU.'\n )\n else:\n raise ValueError(\n 'encryption_spec_key_name (CMEK) is not supported for TPU '\n 'training.'\n )\n else:\n custom_job_payload['encryption_spec'] = {\n 'kms_key_name': encryption_spec_key_name\n }\n\n custom_job_payload_str = json.dumps(custom_job_payload)\n custom_job_payload_list = custom_job_payload_str.rsplit(\n f'\"{executor_line}\"', maxsplit=1\n )\n outputs = NamedTuple(\n 'outputs', payload_first_part=str, payload_second_part=str, location=str\n )\n return outputs(\n payload_first_part=custom_job_payload_list[0],\n payload_second_part=custom_job_payload_list[1],\n location=location,\n )\n except Exception as e: # pylint: disable=broad-exception-caught\n if isinstance(e, ValueError):\n raise\n logging.exception(str(e))\n sys.exit(13)\n\n" | |
| ], | |
| "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1" | |
| } | |
| }, | |
| "exec-create-endpoint-and-deploy-model": { | |
| "container": { | |
| "args": [ | |
| "--executor_input", | |
| "{{$}}", | |
| "--function_to_execute", | |
| "create_endpoint_and_deploy_model" | |
| ], | |
| "command": [ | |
| "sh", | |
| "-ec", | |
| "program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", | |
| "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef create_endpoint_and_deploy_model(\n project: str,\n location: str,\n model_resource_name: str,\n display_name: str,\n regional_endpoint: str,\n endpoint_resource_name: dsl.OutputPath(str),\n create_endpoint_gcp_resources: dsl.OutputPath(str),\n deploy_model_gcp_resources: dsl.OutputPath(str),\n encryption_spec_key_name: str = '',\n service_account: str = '',\n deploy_model: bool = True,\n):\n \"\"\"Creates a vertex endpoint and deploy the specified model.\n\n Args:\n project: Name of the GCP project.\n location: Location for model upload and deployment.\n model_resource_name: Path to the created Model on Model Registry.\n display_name: Name of the model (shown in Model Registry).\n regional_endpoint: Regional API endpoint.\n encryption_spec_key_name: Customer-managed encryption key.\n service_account: If set, then a custom service account will be used.\n deploy_model: Whether to deploy the model to an endpoint. Default\n is ``True``. If ``False``, the model will not be deployed and output\n artifacts will contain empty strings.\n\n Returns:\n endpoint_resource_name: Path to the created endpoint on Online Prediction.\n create_endpoint_gcp_resources: Serialized JSON of GCP resources for\n creating an endpoint.\n deploy_model_gcp_resources: Serialized JSON of GCP resources for deploying\n the model.\n \"\"\"\n import json\n import logging\n import os\n import sys\n from typing import Any, Dict\n\n try:\n from google_cloud_pipeline_components.container.v1.gcp_launcher import lro_remote_runner\n except ImportError:\n from google_cloud_pipeline_components.google_cloud_pipeline_components.container.v1.gcp_launcher import lro_remote_runner\n\n def run_lro_remote_runner(\n url: str, payload: Dict[str, Any], gcp_resources: str) -> Any:\n remote_runner = lro_remote_runner.LroRemoteRunner(location)\n lro = remote_runner.create_lro(\n url, json.dumps(payload), gcp_resources)\n return remote_runner.poll_lro(lro=lro)\n\n try:\n os.makedirs(os.path.dirname(endpoint_resource_name), exist_ok=True)\n\n if not deploy_model:\n with open(endpoint_resource_name, 'w') as fout:\n fout.write('')\n return\n\n regional_endpoint = regional_endpoint.rstrip('/')\n\n create_endpoint_payload = {\n 'displayName': display_name,\n }\n\n pipeline_labels_str = os.getenv('VERTEX_AI_PIPELINES_RUN_LABELS')\n if pipeline_labels_str:\n create_endpoint_payload['labels'] = json.loads(pipeline_labels_str)\n\n if encryption_spec_key_name:\n create_endpoint_payload['encryption_spec'] = {\n 'kms_key_name': encryption_spec_key_name\n }\n\n create_endpoint_lro = run_lro_remote_runner(\n url=(f'{regional_endpoint}/projects/{project}/locations/{location}'\n '/endpoints'),\n payload=create_endpoint_payload,\n gcp_resources=create_endpoint_gcp_resources)\n\n response_endpoint = create_endpoint_lro['response']['name']\n with open(endpoint_resource_name, 'w') as fout:\n fout.write(response_endpoint)\n\n logging.info(\n 'Endpoint created successfully. Deploying model %s to endpoint',\n model_resource_name)\n\n deploy_model_payload = {\n 'deployedModel': {\n 'model': model_resource_name,\n 'displayName': display_name,\n 'automaticResources': {\n 'minReplicaCount': 1,\n 'maxReplicaCount': 1\n }\n }\n }\n if service_account:\n deploy_model_payload['deployedModel']['service_account'] = service_account\n\n _ = run_lro_remote_runner(\n url=f'{regional_endpoint}/{response_endpoint}:deployModel',\n payload=deploy_model_payload,\n gcp_resources=deploy_model_gcp_resources)\n\n logging.info('Model deployed successfully!')\n except Exception as e: # pylint: disable=broad-exception-caught\n if isinstance(e, ValueError):\n raise\n logging.exception(str(e))\n sys.exit(13)\n\n" | |
| ], | |
| "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1" | |
| } | |
| }, | |
| "exec-dataset-encoder": { | |
| "container": { | |
| "args": [ | |
| "--type", | |
| "CustomJob", | |
| "--payload", | |
| "{\"display_name\": \"dataset-encoder\", \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": \"1\", \"machine_spec\": {\"machine_type\": \"{{$.inputs.parameters['machine_type']}}\"}, \"container_spec\": {\"image_uri\": \"{{$.inputs.parameters['image_uri']}}\", \"args\": [\"--dataset_uri={{$.inputs.parameters['dataset_uri']}}\", \"--default_context={{$.inputs.parameters['default_context'].json_escape[0]}}\", \"--large_model_reference={{$.inputs.parameters['large_model_reference']}}\", \"--min_examples={{$.inputs.parameters['min_examples']}}\", \"--max_examples={{$.inputs.parameters['max_examples']}}\", \"--output_dataset_path={{$.outputs.artifacts['encoded_data'].uri}}/output_dataset\", \"--encoded_dataset_path={{$.outputs.parameters['encoded_dataset_path'].output_file}}\", \"--training_dataset_path={{$.outputs.parameters['dataset_path'].output_file}}\", \"--encoded_dataset_stats={{$.outputs.parameters['encoded_dataset_stats'].output_file}}\", \"--executor_input={{$.json_escape[1]}}\"]}}], \"scheduling\": {\"disable_retries\": false}}, \"encryption_spec\": {\"kms_key_name\": \"{{$.inputs.parameters['encryption_spec_key_name']}}\"}, \"labels\": {\"vertex-internal-enable-custom-job-retries\": \"\"}}", | |
| "--project", | |
| "{{$.inputs.parameters['project']}}", | |
| "--location", | |
| "{{$.inputs.parameters['location']}}", | |
| "--gcp_resources", | |
| "{{$.outputs.parameters['gcp_resources'].output_file}}" | |
| ], | |
| "command": [ | |
| "python3", | |
| "-u", | |
| "-m", | |
| "google_cloud_pipeline_components.container.v1.custom_job.launcher" | |
| ], | |
| "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:2.10.0" | |
| } | |
| }, | |
| "exec-dataset-encoder-2": { | |
| "container": { | |
| "args": [ | |
| "--type", | |
| "CustomJob", | |
| "--payload", | |
| "{\"display_name\": \"dataset-encoder\", \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": \"1\", \"machine_spec\": {\"machine_type\": \"{{$.inputs.parameters['machine_type']}}\"}, \"container_spec\": {\"image_uri\": \"{{$.inputs.parameters['image_uri']}}\", \"args\": [\"--dataset_uri={{$.inputs.parameters['dataset_uri']}}\", \"--default_context={{$.inputs.parameters['default_context'].json_escape[0]}}\", \"--large_model_reference={{$.inputs.parameters['large_model_reference']}}\", \"--min_examples={{$.inputs.parameters['min_examples']}}\", \"--max_examples={{$.inputs.parameters['max_examples']}}\", \"--output_dataset_path={{$.outputs.artifacts['encoded_data'].uri}}/output_dataset\", \"--encoded_dataset_path={{$.outputs.parameters['encoded_dataset_path'].output_file}}\", \"--training_dataset_path={{$.outputs.parameters['dataset_path'].output_file}}\", \"--encoded_dataset_stats={{$.outputs.parameters['encoded_dataset_stats'].output_file}}\", \"--executor_input={{$.json_escape[1]}}\"]}}], \"scheduling\": {\"disable_retries\": false}}, \"encryption_spec\": {\"kms_key_name\": \"{{$.inputs.parameters['encryption_spec_key_name']}}\"}, \"labels\": {\"vertex-internal-enable-custom-job-retries\": \"\"}}", | |
| "--project", | |
| "{{$.inputs.parameters['project']}}", | |
| "--location", | |
| "{{$.inputs.parameters['location']}}", | |
| "--gcp_resources", | |
| "{{$.outputs.parameters['gcp_resources'].output_file}}" | |
| ], | |
| "command": [ | |
| "python3", | |
| "-u", | |
| "-m", | |
| "google_cloud_pipeline_components.container.v1.custom_job.launcher" | |
| ], | |
| "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:2.10.0" | |
| } | |
| }, | |
| "exec-dataset-encoder-3": { | |
| "container": { | |
| "args": [ | |
| "--type", | |
| "CustomJob", | |
| "--payload", | |
| "{\"display_name\": \"dataset-encoder\", \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": \"1\", \"machine_spec\": {\"machine_type\": \"{{$.inputs.parameters['machine_type']}}\"}, \"container_spec\": {\"image_uri\": \"{{$.inputs.parameters['image_uri']}}\", \"args\": [\"--dataset_uri={{$.inputs.parameters['dataset_uri']}}\", \"--default_context={{$.inputs.parameters['default_context'].json_escape[0]}}\", \"--large_model_reference={{$.inputs.parameters['large_model_reference']}}\", \"--min_examples={{$.inputs.parameters['min_examples']}}\", \"--max_examples={{$.inputs.parameters['max_examples']}}\", \"--output_dataset_path={{$.outputs.artifacts['encoded_data'].uri}}/output_dataset\", \"--encoded_dataset_path={{$.outputs.parameters['encoded_dataset_path'].output_file}}\", \"--training_dataset_path={{$.outputs.parameters['dataset_path'].output_file}}\", \"--encoded_dataset_stats={{$.outputs.parameters['encoded_dataset_stats'].output_file}}\", \"--executor_input={{$.json_escape[1]}}\"]}}], \"scheduling\": {\"disable_retries\": false}}, \"encryption_spec\": {\"kms_key_name\": \"{{$.inputs.parameters['encryption_spec_key_name']}}\"}, \"labels\": {\"vertex-internal-enable-custom-job-retries\": \"\"}}", | |
| "--project", | |
| "{{$.inputs.parameters['project']}}", | |
| "--location", | |
| "{{$.inputs.parameters['location']}}", | |
| "--gcp_resources", | |
| "{{$.outputs.parameters['gcp_resources'].output_file}}" | |
| ], | |
| "command": [ | |
| "python3", | |
| "-u", | |
| "-m", | |
| "google_cloud_pipeline_components.container.v1.custom_job.launcher" | |
| ], | |
| "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:2.10.0" | |
| } | |
| }, | |
| "exec-dataset-encoder-4": { | |
| "container": { | |
| "args": [ | |
| "--type", | |
| "CustomJob", | |
| "--payload", | |
| "{\"display_name\": \"dataset-encoder\", \"job_spec\": {\"worker_pool_specs\": [{\"replica_count\": \"1\", \"machine_spec\": {\"machine_type\": \"{{$.inputs.parameters['machine_type']}}\"}, \"container_spec\": {\"image_uri\": \"{{$.inputs.parameters['image_uri']}}\", \"args\": [\"--dataset_uri={{$.inputs.parameters['dataset_uri']}}\", \"--default_context={{$.inputs.parameters['default_context'].json_escape[0]}}\", \"--large_model_reference={{$.inputs.parameters['large_model_reference']}}\", \"--min_examples={{$.inputs.parameters['min_examples']}}\", \"--max_examples={{$.inputs.parameters['max_examples']}}\", \"--output_dataset_path={{$.outputs.artifacts['encoded_data'].uri}}/output_dataset\", \"--encoded_dataset_path={{$.outputs.parameters['encoded_dataset_path'].output_file}}\", \"--training_dataset_path={{$.outputs.parameters['dataset_path'].output_file}}\", \"--encoded_dataset_stats={{$.outputs.parameters['encoded_dataset_stats'].output_file}}\", \"--executor_input={{$.json_escape[1]}}\"]}}], \"scheduling\": {\"disable_retries\": false}}, \"encryption_spec\": {\"kms_key_name\": \"{{$.inputs.parameters['encryption_spec_key_name']}}\"}, \"labels\": {\"vertex-internal-enable-custom-job-retries\": \"\"}}", | |
| "--project", | |
| "{{$.inputs.parameters['project']}}", | |
| "--location", | |
| "{{$.inputs.parameters['location']}}", | |
| "--gcp_resources", | |
| "{{$.outputs.parameters['gcp_resources'].output_file}}" | |
| ], | |
| "command": [ | |
| "python3", | |
| "-u", | |
| "-m", | |
| "google_cloud_pipeline_components.container.v1.custom_job.launcher" | |
| ], | |
| "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:2.10.0" | |
| } | |
| }, | |
| "exec-dummy-prompt-validation-component": { | |
| "container": { | |
| "args": [ | |
| "--executor_input", | |
| "{{$}}", | |
| "--function_to_execute", | |
| "dummy_prompt_validation_component" | |
| ], | |
| "command": [ | |
| "sh", | |
| "-ec", | |
| "program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", | |
| "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef dummy_prompt_validation_component(\n encoded_dataset_stats: Optional[dict] = None\n) -> dict:\n return encoded_dataset_stats\n\n" | |
| ], | |
| "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1" | |
| } | |
| }, | |
| "exec-export-managed-dataset": { | |
| "container": { | |
| "command": [ | |
| "sh", | |
| "-c", | |
| "set -e -x\nDATASET_NAME=\"$0\"\nDATASET_URI=\"$1\"\nREGIONAL_ENDPOINT=\"$2\"\nEXPORT_LOCATION=\"$3\"\n\nif [ -z \"${DATASET_URI}\" ] && [ -z \"${DATASET_NAME}\" ]\nthen\n echo \"One of dataset_uri and dataset_name must be specified.\"\n exit 1\nfi\n\nif [ -n \"${DATASET_URI}\" ] && [ -n \"${DATASET_NAME}\" ]\nthen\n echo \"Only one of dataset_uri and dataset_name may be set.\"\n exit 1\nfi\n\n# Construct export output dir; the file containing the dataset location is also located here.\nEXPORT_DIR=\"$(dirname \"$EXPORT_LOCATION\")\"\nmkdir -p $EXPORT_DIR\nEXPORT_DIR_GCS=$(echo $EXPORT_DIR | sed 's/^\\/gcs\\//gs:\\/\\//g')\n\nif [ -n \"${DATASET_URI}\" ]\nthen\n echo \"Dataset export not required for an unmanaged file.\"\n echo -n $DATASET_URI > \"$EXPORT_LOCATION\"\n exit 0\nfi\n\nEXPORT_LRO=$(curl -X POST \\\n -H \"Authorization: Bearer $(gcloud auth print-access-token)\" \\\n -H \"Content-Type: application/json\" \\\n \"${REGIONAL_ENDPOINT}/${DATASET_NAME}:export\" \\\n -d \"{\n 'exportConfig': {\n 'gcsDestination': {\n 'outputUriPrefix': '$EXPORT_DIR_GCS'\n },\n 'exportUse': 'ADAPTER_TUNING'\n }\n }\")\nLRO_NAME=$(echo $EXPORT_LRO | python3 -c \"import sys, json; print(json.load(sys.stdin)['name'])\")\nPOLL_ENDPOINT=\"${REGIONAL_ENDPOINT}/${LRO_NAME}\"\n\nwhile true\ndo\n POLL=$(curl -X GET -H \"Authorization: Bearer $(gcloud auth print-access-token)\" -H \"Content-Type: application/json\" $POLL_ENDPOINT)\n echo $POLL\n STATUS=$(echo $POLL | python3 -c \"import sys, json; print(json.load(sys.stdin)['metadata']['genericMetadata']['state'])\")\n if [ \"$STATUS\" = \"DONE\" ]\n then\n ERROR=$(echo $POLL | python3 -c \"import sys, json; from collections import defaultdict; d = defaultdict(str); d.update(json.load(sys.stdin)); print(d['error'])\")\n if [ ${#ERROR} -gt 0 ]\n then\n exit 13\n fi\n RESPONSE_EXPORTED_FILE=$(echo $POLL | python3 -c \"import sys, json; from collections import defaultdict; d = defaultdict(str); d.update(json.load(sys.stdin)); print(d['response']['exportedFiles'][0])\")\n echo -n $RESPONSE_EXPORTED_FILE > \"$EXPORT_LOCATION\"\n exit 0\n fi\n sleep 10\ndone\n", | |
| "{{$.inputs.parameters['dataset_name']}}", | |
| "{{$.inputs.parameters['dataset_uri']}}", | |
| "{{$.inputs.parameters['regional_endpoint']}}", | |
| "{{$.outputs.parameters['export_location'].output_file}}" | |
| ], | |
| "image": "gcr.io/google.com/cloudsdktool/cloud-sdk:latest" | |
| } | |
| }, | |
| "exec-large-language-model-tuner": { | |
| "container": { | |
| "args": [ | |
| "--type", | |
| "CustomJob", | |
| "--project", | |
| "{{$.inputs.parameters['project']}}", | |
| "--location", | |
| "{{$.inputs.parameters['location']}}", | |
| "--gcp_resources", | |
| "{{$.outputs.parameters['gcp_resources'].output_file}}", | |
| "--payload", | |
| "{\"Concat\": [\"{{$.inputs.parameters['payload_first_part']}}\", \"\\\"--executor_input={{$.json_escape[1]}}\\\"\", \"{{$.inputs.parameters['payload_second_part']}}\"]}" | |
| ], | |
| "command": [ | |
| "python3", | |
| "-u", | |
| "-m", | |
| "google_cloud_pipeline_components.container.v1.custom_job.launcher" | |
| ], | |
| "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:1.0.24" | |
| } | |
| }, | |
| "exec-tensorboard-uploader": { | |
| "container": { | |
| "command": [ | |
| "bash", | |
| "-c", | |
| "set -e -x\nPROJECT=\"$0\"\nLOCATION=\"$1\"\nTENSORBOARD_ID=\"$2\"\nMETRICS_DIRECTORY_URI=\"$3\"\nEXPERIMENT_NAME=\"$4\"\nTENSORBOARD_URI=\"$5\"\n\nmkdir -p \"$(dirname ${TENSORBOARD_URI})\"\nif [ -z \"${TENSORBOARD_ID}\" ];\nthen\n echo \"TensorBoard ID is not set. Skip uploading the TensorBoard.\"\n echo -n \"\" > \"${TENSORBOARD_URI}\"\n exit 0\nfi\n\nif [ -z \"${PROJECT}\" ]; then\n echo \"Project is not set.\"\n exit 1\nelif [ -z \"${LOCATION}\" ]; then\n echo \"Location is not set.\"\n exit 1\nelif [ -z \"${METRICS_DIRECTORY_URI}\" ]; then\n echo \"Metrics directory uri is not set.\"\n exit 1\nelif [ -z \"${EXPERIMENT_NAME}\" ]; then\n echo \"Experiment name is not set.\"\n exit 1\nelif [ -z \"${TENSORBOARD_URI}\" ]; then\n echo \"TensorBoard URI is not set.\"\n exit 1\nfi\n\ncase \"${METRICS_DIRECTORY_URI}\" in\n \"gs://\"*) ;;\n *)\n echo \"Invalid metrics directory uri. Metrics directory uri must start with gs://.\"\n exit 1\n ;;\nesac\n\nif [[ \"${TENSORBOARD_ID}\" =~ ^[0-9]+$ ]]; then\n TENSORBOARD_RESOURCE_ID=\"projects/${PROJECT}/locations/${LOCATION}/tensorboards/${TENSORBOARD_ID}\"\nelif [[ \"${TENSORBOARD_ID}\" =~ ^projects/[^/]+/locations/[^/]+/tensorboards/[0-9]+$ ]]; then\n echo \"Split tensorboard id\"\n TENSORBOARD_RESOURCE_ID=\"${TENSORBOARD_ID}\"\n TENSORBOARD_RESOURCE_ARR=(${TENSORBOARD_ID//\\// })\n PROJECT=${TENSORBOARD_RESOURCE_ARR[1]}\n LOCATION=${TENSORBOARD_RESOURCE_ARR[3]}\n TENSORBOARD_ID=${TENSORBOARD_RESOURCE_ARR[5]}\nelse\n echo '[ERROR]: Invalid format of tensorboard_id. It should be either a '\\\n'string of digits or projects/${PROJECT}/location/${LOCATION}/'\\\n'tensorboards/${TENSORBOARD_ID_DIGITS}'\n exit 1\nfi\n\nset +e\n\n/opt/conda/bin/tb-gcp-uploader --tensorboard_resource_name \\\n \"${TENSORBOARD_RESOURCE_ID}\" \\\n --logdir=\"${METRICS_DIRECTORY_URI}\" \\\n --experiment_name=\"${EXPERIMENT_NAME}\" \\\n --one_shot=True\n\nif [ $? -ne 0 ]; then\n exit 13\nfi\n\nset -e\n\nweb_server_uri=\"tensorboard.googleusercontent.com\"\ntensorboard_resource_name_uri=\"projects+${PROJECT}+locations+${LOCATION}+tensorboards+${TENSORBOARD_ID}+experiments+${EXPERIMENT_NAME}\"\necho -n \"https://${LOCATION}.${web_server_uri}/experiment/${tensorboard_resource_name_uri}\" > \"${TENSORBOARD_URI}\"\n", | |
| "{{$.inputs.parameters['project']}}", | |
| "{{$.inputs.parameters['location']}}", | |
| "{{$.inputs.parameters['tensorboard_id']}}", | |
| "{{$.inputs.artifacts['metrics_directory_uri'].uri}}", | |
| "{{$.inputs.parameters['experiment_name']}}", | |
| "{{$.outputs.parameters['tensorboard_uri'].output_file}}" | |
| ], | |
| "image": "us-docker.pkg.dev/vertex-ai/training/tf-cpu.2-11:latest" | |
| } | |
| }, | |
| "exec-upload-llm-model": { | |
| "container": { | |
| "args": [ | |
| "--executor_input", | |
| "{{$}}", | |
| "--function_to_execute", | |
| "upload_llm_model" | |
| ], | |
| "command": [ | |
| "sh", | |
| "-ec", | |
| "program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", | |
| "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef upload_llm_model(\n project: str,\n location: str,\n artifact_uri: dsl.Input[dsl.Artifact],\n model_reference_name: str,\n model_display_name: str,\n regional_endpoint: str,\n model_resource_name: dsl.OutputPath(str),\n gcp_resources: dsl.OutputPath(str),\n encryption_spec_key_name: str = '',\n upload_model: bool = True,\n tune_type: str = '',\n):\n \"\"\"Uploads LLM model.\n\n Args:\n project: Name of the GCP project.\n location: Location for model upload and deployment.\n artifact_uri: KFP Artifact for adapter.\n model_reference_name: Large model reference name.\n model_display_name: Name of the model (shown in Model Registry).\n regional_endpoint: Regional API endpoint.\n encryption_spec_key_name: Customer-managed encryption key.\n upload_model: Whether to upload the model to the Model Registry. Default\n is ``True``. If ``False``, the model will not be uploaded and output\n artifacts will contain empty strings.\n tune_type: Method used to tune the model, e.g. ``distillation``. If\n present, this value is used to set the ``tune-type`` run label during\n model upload.\n\n Returns:\n model_resource_name: Path to the created Model on Model Registry.\n gcp_resources: Serialized JSON of `gcp_resources`.\n \"\"\"\n import json\n import logging\n import os\n import sys\n try:\n from google_cloud_pipeline_components.container.v1.gcp_launcher import lro_remote_runner\n except ImportError:\n from google_cloud_pipeline_components.google_cloud_pipeline_components.container.v1.gcp_launcher import lro_remote_runner\n\n try:\n os.makedirs(os.path.dirname(model_resource_name), exist_ok=True)\n\n if not upload_model:\n with open(model_resource_name, 'w') as fout:\n fout.write('')\n return\n\n pipeline_labels_str = os.getenv('VERTEX_AI_PIPELINES_RUN_LABELS')\n labels = (\n json.loads(pipeline_labels_str) if pipeline_labels_str else {})\n labels['google-vertex-llm-tuning-base-model-id'] = (\n model_reference_name.replace('@', '-')\n )\n if tune_type:\n labels['tune-type'] = tune_type\n\n model_upload_payload = {\n 'model': {\n 'displayName': model_display_name,\n 'largeModelReference': {\n 'name': model_reference_name\n },\n 'labels': labels,\n 'generatedModelSource': {\n 'genie_source': {\n 'base_model_uri': ''\n }\n },\n 'artifactUri': artifact_uri.uri\n }\n }\n if encryption_spec_key_name:\n model_upload_payload['model']['encryption_spec'] = {\n 'kms_key_name': encryption_spec_key_name\n }\n\n regional_endpoint = regional_endpoint.rstrip('/')\n upload_model_uri = (\n f'{regional_endpoint}/projects/{project}/locations/{location}/models:'\n 'upload')\n\n remote_runner = lro_remote_runner.LroRemoteRunner(location)\n upload_model_lro = remote_runner.create_lro(\n upload_model_uri,\n json.dumps(model_upload_payload),\n gcp_resources,\n )\n upload_model_lro = remote_runner.poll_lro(lro=upload_model_lro)\n model_resource = upload_model_lro['response']['model']\n model_version_id = (\n upload_model_lro['response'].get('model_version_id')\n or upload_model_lro['response'].get('modelVersionId'))\n if model_version_id:\n model_resource += f'@{model_version_id}'\n\n with open(model_resource_name, 'w') as fout:\n fout.write(model_resource)\n\n except Exception as e: # pylint: disable=broad-exception-caught\n if isinstance(e, ValueError):\n raise\n logging.exception(str(e))\n sys.exit(13)\n\n" | |
| ], | |
| "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1" | |
| } | |
| }, | |
| "exec-validate-pipeline": { | |
| "container": { | |
| "args": [ | |
| "--executor_input", | |
| "{{$}}", | |
| "--function_to_execute", | |
| "validate_pipeline" | |
| ], | |
| "command": [ | |
| "sh", | |
| "-ec", | |
| "program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" > \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n", | |
| "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef validate_pipeline(\n location: str = '{{$.pipeline_google_cloud_location}}',\n model: str = 'text-bison-001',\n encryption_spec_key_name: str = '',\n tpu_training_skip_cmek: bool = False,\n api_endpoint: str = 'aiplatform.googleapis.com/ui',\n tuning_method: str = 'tune',\n evaluation_data_uri: str = '',\n evaluation_output_root_dir: str = '',\n tensorboard_resource_id: str = '',\n accelerator_type: str = '',\n pipeline_region: str = '{{$.pipeline_google_cloud_location}}',\n max_context_length: str = '',\n enable_tpu_v5e: bool = False,\n) -> NamedTuple('outputs',\n model_dash_version=str,\n model_at_version=str,\n use_tpu=bool,\n adapter_training_location=str,\n with_tensorboard=bool,\n with_batch_prediction_and_evaluation=bool,\n service_account=str,\n regional_endpoint=str,\n model_location=str,\n use_v2_preprocessing=bool):\n \"\"\"Validate and preprocess pipeline parameters.\n\n Args:\n location: Region in which all the components except for tuning job should\n run.\n model: Large language reference model to use (default: `text-bison-001`).\n encryption_spec_key_name: If set, CMEK will be used.\n tpu_training_skip_cmek: If True, skip CMEK setup for TPU training.\n api_endpoint: API endpoint.\n tuning_method: Tuning method.\n evaluation_data_uri: GCS URI of eval dataset. If evaluation dataset is\n provided, the best tuned checkpoint will be selected.\n evaluation_output_root_dir: Root GCS URI of evaluation outputs.\n tensorboard_resource_id: The Vertex AI TensorBoard Resource/Instance ID.\n If this parameter is set, then a Vertex AI TensorBoard Experiment will\n be launched.\n accelerator_type: One of 'TPU' or 'GPU'. If 'TPU' is specified, tuning\n runs in europe-west4, else in us-central1.\n pipeline_region: The region the pipeline runs in.\n max_context_length: The max context length used for tuning. Can be either\n '8k' or '32k'.\n enable_tpu_v5e: Whether to enable TPU V5e tuning or not.\n\n Returns:\n outputs: NamedTuple consisting of fields of model_dash_version,\n model_at_version, use_tpu, adapter_training_location,\n with_tensorboard, with_batch_prediction_and_evaluation,\n service_account, regional_endpoint, model_location,\n use_v2_preprocessing.\n \"\"\"\n import os\n import logging\n import sys\n\n try:\n if not accelerator_type and 'us-central1' != location:\n raise ValueError('Tuning is only supported in us-central1.')\n\n # Convert model version with formats text-bison-001 and text-bison@001.\n if tuning_method == 'tune' and (\n 'text-bison-001' == model or 'text-bison@001' == model\n ):\n model_dash_version = 'text-bison-001'\n model_at_version = 'text-bison@001'\n else:\n model_dash_version = model\n model_at_version = model\n\n service_account = os.environ['CLOUD_ML_JOB_SA']\n\n if max_context_length:\n if model != 'gemini-pro':\n raise ValueError(f'max_context_length is not support for {model}')\n if max_context_length.lower() not in ['8k', '32k']:\n raise ValueError(f'Invalid value for max_context_length:'\n f' {max_context_length}, only 8k or 32k are supported')\n\n supported_pipeline_regions = [\n 'europe-west4',\n 'asia-southeast1',\n 'us-west1',\n 'europe-west3',\n 'europe-west2',\n 'asia-northeast1',\n 'us-central1',\n 'us-east4',\n 'us-west4',\n 'northamerica-northeast1',\n 'europe-west9',\n 'europe-west1',\n 'asia-northeast3',\n ]\n if pipeline_region not in supported_pipeline_regions:\n raise ValueError(\n f'Unsupported pipeline region: {pipeline_region}. Must be one of'\n f' {supported_pipeline_regions}.'\n )\n\n if accelerator_type:\n if accelerator_type.upper() == 'TPU':\n use_tpu = True\n if not pipeline_region.startswith('europe') and not (\n max_context_length == '32k' or '32k' in model\n ) and enable_tpu_v5e:\n adapter_training_location = 'us-west1'\n else:\n adapter_training_location = 'europe-west4'\n elif accelerator_type.upper() == 'GPU':\n use_tpu = False\n adapter_training_location = 'us-central1'\n else:\n raise ValueError(\n f'Unsupported accelerator_type: {accelerator_type}. Should'\n ' be one of \"TPU\" or \"GPU\".'\n )\n else:\n if 'europe-west4' == pipeline_region:\n use_tpu = True\n adapter_training_location = 'europe-west4'\n elif 'us-central1' == pipeline_region:\n use_tpu = False\n adapter_training_location = 'us-central1'\n else:\n raise ValueError(\n '`accelerator_type` must be specified to run a pipeline in'\n f' {pipeline_region}.'\n )\n\n location = pipeline_region if not location else location\n\n if use_tpu:\n if encryption_spec_key_name:\n if tpu_training_skip_cmek:\n logging.warning(\n 'CMEK will not be set for TPU training. If this is not expected, '\n 'please run with GPU.'\n )\n else:\n raise ValueError(\n 'encryption_spec_key_name (CMEK) is not supported for TPU '\n 'training at the moment. Please either unset '\n 'encryption_spec_key_name or create your pipeline in us-central1 '\n 'to use GPU instead.'\n )\n\n if not use_tpu:\n if encryption_spec_key_name:\n if location != 'us-central1' or pipeline_region != 'us-central1':\n raise ValueError(\n 'encryption_spec_key_name (CMEK) for GPU training is only'\n ' supported in us-central1. Please either unset'\n ' encryption_spec_key_name or create your pipeline in and/or'\n ' specifythe model upload location to be us-central1 instead.'\n )\n if model.startswith('gemini'):\n raise ValueError(\n 'GPU is not supported for gemini model. Please set accelerator_type'\n ' to TPU, or run pipeline in europe-west4.'\n )\n\n outputs = NamedTuple(\n 'outputs',\n model_dash_version=str,\n model_at_version=str,\n use_tpu=bool,\n adapter_training_location=str,\n with_tensorboard=bool,\n with_batch_prediction_and_evaluation=bool,\n service_account=str,\n regional_endpoint=str,\n model_location=str,\n use_v2_preprocessing=bool,\n )\n ret = outputs(\n model_dash_version=model_dash_version,\n model_at_version=model_at_version,\n use_tpu=use_tpu,\n adapter_training_location=adapter_training_location,\n with_tensorboard=bool(\n 'tune' != tuning_method and tensorboard_resource_id),\n with_batch_prediction_and_evaluation=bool(\n evaluation_data_uri and evaluation_output_root_dir),\n service_account=service_account,\n regional_endpoint=f'https://{location}-{api_endpoint}',\n model_location=location,\n use_v2_preprocessing=model.startswith('gemini'),\n )\n logging.info('use_tpu = [%s]', ret.use_tpu)\n return ret\n except Exception as e: # pylint: disable=broad-exception-caught\n if isinstance(e, ValueError):\n raise\n logging.exception(str(e))\n sys.exit(13)\n\n" | |
| ], | |
| "image": "gcr.io/ml-pipeline/google-cloud-pipeline-components:2.3.1" | |
| } | |
| } | |
| } | |
| }, | |
| "pipelineInfo": { | |
| "description": "Runs adapter training pipeline.", | |
| "name": "tune-large-model" | |
| }, | |
| "root": { | |
| "dag": { | |
| "outputs": { | |
| "parameters": { | |
| "endpoint_resource_name": { | |
| "valueFromParameter": { | |
| "outputParameterKey": "endpoint_resource_name", | |
| "producerSubtask": "deployment-graph" | |
| } | |
| }, | |
| "model_resource_name": { | |
| "valueFromParameter": { | |
| "outputParameterKey": "model_resource_name", | |
| "producerSubtask": "deployment-graph" | |
| } | |
| } | |
| } | |
| }, | |
| "tasks": { | |
| "condition-1": { | |
| "componentRef": { | |
| "name": "comp-condition-1" | |
| }, | |
| "dependentTasks": [ | |
| "tuning-graph", | |
| "validate-pipeline" | |
| ], | |
| "inputs": { | |
| "artifacts": { | |
| "pipelinechannel--tuning-graph-tensorboard_metrics": { | |
| "taskOutputArtifact": { | |
| "outputArtifactKey": "tensorboard_metrics", | |
| "producerTask": "tuning-graph" | |
| } | |
| } | |
| }, | |
| "parameters": { | |
| "pipelinechannel--project": { | |
| "componentInputParameter": "project" | |
| }, | |
| "pipelinechannel--tensorboard_resource_id": { | |
| "componentInputParameter": "tensorboard_resource_id" | |
| }, | |
| "pipelinechannel--validate-pipeline-with_tensorboard": { | |
| "taskOutputParameter": { | |
| "outputParameterKey": "with_tensorboard", | |
| "producerTask": "validate-pipeline" | |
| } | |
| } | |
| } | |
| }, | |
| "taskInfo": { | |
| "name": "tensorboard-uploader-condition" | |
| }, | |
| "triggerPolicy": { | |
| "condition": "inputs.parameter_values['pipelinechannel--validate-pipeline-with_tensorboard'] == true" | |
| } | |
| }, | |
| "deployment-graph": { | |
| "cachingOptions": { | |
| "enableCache": true | |
| }, | |
| "componentRef": { | |
| "name": "comp-deployment-graph" | |
| }, | |
| "dependentTasks": [ | |
| "tuning-graph", | |
| "validate-pipeline" | |
| ], | |
| "inputs": { | |
| "artifacts": { | |
| "saved_model": { | |
| "taskOutputArtifact": { | |
| "outputArtifactKey": "saved_model", | |
| "producerTask": "tuning-graph" | |
| } | |
| } | |
| }, | |
| "parameters": { | |
| "deploy_model": { | |
| "componentInputParameter": "deploy_model" | |
| }, | |
| "encryption_spec_key_name": { | |
| "componentInputParameter": "encryption_spec_key_name" | |
| }, | |
| "large_model_reference": { | |
| "taskOutputParameter": { | |
| "outputParameterKey": "model_at_version", | |
| "producerTask": "validate-pipeline" | |
| } | |
| }, | |
| "model_display_name": { | |
| "componentInputParameter": "model_display_name" | |
| }, | |
| "model_location": { | |
| "taskOutputParameter": { | |
| "outputParameterKey": "model_location", | |
| "producerTask": "validate-pipeline" | |
| } | |
| }, | |
| "project": { | |
| "componentInputParameter": "project" | |
| }, | |
| "regional_endpoint": { | |
| "taskOutputParameter": { | |
| "outputParameterKey": "regional_endpoint", | |
| "producerTask": "validate-pipeline" | |
| } | |
| }, | |
| "service_account": { | |
| "taskOutputParameter": { | |
| "outputParameterKey": "service_account", | |
| "producerTask": "validate-pipeline" | |
| } | |
| } | |
| } | |
| }, | |
| "taskInfo": { | |
| "name": "deployment-graph" | |
| } | |
| }, | |
| "tuning-graph": { | |
| "cachingOptions": { | |
| "enableCache": true | |
| }, | |
| "componentRef": { | |
| "name": "comp-tuning-graph" | |
| }, | |
| "dependentTasks": [ | |
| "validate-pipeline" | |
| ], | |
| "inputs": { | |
| "parameters": { | |
| "adapter_training_location": { | |
| "taskOutputParameter": { | |
| "outputParameterKey": "adapter_training_location", | |
| "producerTask": "validate-pipeline" | |
| } | |
| }, | |
| "dataset_name": { | |
| "componentInputParameter": "dataset_name" | |
| }, | |
| "dataset_uri": { | |
| "componentInputParameter": "dataset_uri" | |
| }, | |
| "enable_checkpoint_selection": { | |
| "componentInputParameter": "enable_checkpoint_selection" | |
| }, | |
| "enable_early_stopping": { | |
| "componentInputParameter": "enable_early_stopping" | |
| }, | |
| "encryption_spec_key_name": { | |
| "componentInputParameter": "encryption_spec_key_name" | |
| }, | |
| "evaluation_data_uri": { | |
| "componentInputParameter": "evaluation_data_uri" | |
| }, | |
| "evaluation_interval": { | |
| "componentInputParameter": "evaluation_interval" | |
| }, | |
| "learning_rate": { | |
| "componentInputParameter": "learning_rate" | |
| }, | |
| "learning_rate_multiplier": { | |
| "componentInputParameter": "learning_rate_multiplier" | |
| }, | |
| "max_context_length": { | |
| "componentInputParameter": "max_context_length" | |
| }, | |
| "model": { | |
| "taskOutputParameter": { | |
| "outputParameterKey": "model_dash_version", | |
| "producerTask": "validate-pipeline" | |
| } | |
| }, | |
| "project": { | |
| "componentInputParameter": "project" | |
| }, | |
| "regional_endpoint": { | |
| "taskOutputParameter": { | |
| "outputParameterKey": "regional_endpoint", | |
| "producerTask": "validate-pipeline" | |
| } | |
| }, | |
| "tpu_training_skip_cmek": { | |
| "componentInputParameter": "tpu_training_skip_cmek" | |
| }, | |
| "train_steps": { | |
| "componentInputParameter": "train_steps" | |
| }, | |
| "tuning_method": { | |
| "componentInputParameter": "tuning_method" | |
| }, | |
| "use_tpu": { | |
| "taskOutputParameter": { | |
| "outputParameterKey": "use_tpu", | |
| "producerTask": "validate-pipeline" | |
| } | |
| }, | |
| "use_v2_preprocessing": { | |
| "taskOutputParameter": { | |
| "outputParameterKey": "use_v2_preprocessing", | |
| "producerTask": "validate-pipeline" | |
| } | |
| } | |
| } | |
| }, | |
| "taskInfo": { | |
| "name": "tuning-graph" | |
| } | |
| }, | |
| "validate-pipeline": { | |
| "cachingOptions": {}, | |
| "componentRef": { | |
| "name": "comp-validate-pipeline" | |
| }, | |
| "inputs": { | |
| "parameters": { | |
| "accelerator_type": { | |
| "componentInputParameter": "accelerator_type" | |
| }, | |
| "api_endpoint": { | |
| "componentInputParameter": "api_endpoint" | |
| }, | |
| "encryption_spec_key_name": { | |
| "componentInputParameter": "encryption_spec_key_name" | |
| }, | |
| "evaluation_data_uri": { | |
| "componentInputParameter": "evaluation_data_uri" | |
| }, | |
| "location": { | |
| "componentInputParameter": "location" | |
| }, | |
| "max_context_length": { | |
| "componentInputParameter": "max_context_length" | |
| }, | |
| "model": { | |
| "componentInputParameter": "large_model_reference" | |
| }, | |
| "tensorboard_resource_id": { | |
| "componentInputParameter": "tensorboard_resource_id" | |
| }, | |
| "tpu_training_skip_cmek": { | |
| "componentInputParameter": "tpu_training_skip_cmek" | |
| }, | |
| "tuning_method": { | |
| "componentInputParameter": "tuning_method" | |
| } | |
| } | |
| }, | |
| "taskInfo": { | |
| "name": "validate-pipeline" | |
| } | |
| } | |
| } | |
| }, | |
| "inputDefinitions": { | |
| "parameters": { | |
| "accelerator_type": { | |
| "defaultValue": "GPU", | |
| "description": "One of 'TPU' or 'GPU'. If 'TPU' is specified, tuning\nruns in europe-west4, else in us-central1. Default is 'GPU'.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "api_endpoint": { | |
| "defaultValue": "aiplatform.googleapis.com/ui", | |
| "description": "API endpoint.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "dataset_name": { | |
| "defaultValue": "", | |
| "description": "Name of training dataset if importing from managed dataset.\nOnly this or `dataset_uri` should be set.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "dataset_uri": { | |
| "defaultValue": "", | |
| "description": "GCS URI of training dataset. Only this or `dataset_name`\nshould be set.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "deploy_model": { | |
| "defaultValue": true, | |
| "description": "Whether to create the endpoint and deploy the model.", | |
| "isOptional": true, | |
| "parameterType": "BOOLEAN" | |
| }, | |
| "enable_checkpoint_selection": { | |
| "defaultValue": "default", | |
| "description": "One of 'true', 'false', or 'default'. If set\nto 'true' and if evaluation_data_uri is set, select the best tuned\ncheckpoint. If 'default' is set, checkpoint selection is enabled\nfor bison@001 models, and disabled for others.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "enable_early_stopping": { | |
| "defaultValue": true, | |
| "description": "If set to True, and if evaluation_data_uri is set,\nthen early stopping will be enabled. If early stopping is enabled, the\nmax recommended train_steps is 1,000.", | |
| "isOptional": true, | |
| "parameterType": "BOOLEAN" | |
| }, | |
| "encryption_spec_key_name": { | |
| "defaultValue": "", | |
| "description": "Customer-managed encryption key.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "evaluation_data_uri": { | |
| "defaultValue": "", | |
| "description": "GCS URI of eval dataset.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "evaluation_interval": { | |
| "defaultValue": 20, | |
| "description": "Evaluation after every this number of tuning steps.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_INTEGER" | |
| }, | |
| "evaluation_output_root_dir": { | |
| "defaultValue": "", | |
| "description": "Root GCS URI of evaluation outputs.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "large_model_reference": { | |
| "defaultValue": "text-bison@001", | |
| "description": "Large model reference ID indicating which model to\nuse.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "learning_rate": { | |
| "defaultValue": -1, | |
| "description": "Learning rate hyperparameter for tuning. If this value is\nnegative, then the default learning rate per model per tuning_method\nwill be used. This parameter will be deprecated.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_DOUBLE" | |
| }, | |
| "learning_rate_multiplier": { | |
| "defaultValue": 1, | |
| "description": "Learning rate multiplier for tuning. Under the\nhood, the actual \"learning rate\" will be learning_rate_multiplier *\nthe \"recommended learning rate\" per model, so customers don't have to\nkeep track of the recommended learning rate for each model or tuning\nmethod.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_DOUBLE" | |
| }, | |
| "location": { | |
| "defaultValue": "", | |
| "description": "Location for model upload and deployment.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "max_context_length": { | |
| "defaultValue": "", | |
| "description": "The max context length used for tuning. Can be either\n'8k' or '32k'.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "model_display_name": { | |
| "description": "Name of the model (shown in model registry).", | |
| "parameterType": "STRING" | |
| }, | |
| "project": { | |
| "description": "Name of the GCP project.", | |
| "parameterType": "STRING" | |
| }, | |
| "tensorboard_resource_id": { | |
| "defaultValue": "", | |
| "description": "The Vertex AI TensorBoard Resource/Instance ID.\nIf this parameter is set, then a Vertex AI TensorBoard Experiment will\nbe launched.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| }, | |
| "tpu_training_skip_cmek": { | |
| "defaultValue": false, | |
| "description": "If set to True, CMEK will be ignored during TPU\ntraining step.", | |
| "isOptional": true, | |
| "parameterType": "BOOLEAN" | |
| }, | |
| "train_steps": { | |
| "defaultValue": 300, | |
| "description": "Number of training steps for tuning step.", | |
| "isOptional": true, | |
| "parameterType": "NUMBER_INTEGER" | |
| }, | |
| "tuning_method": { | |
| "defaultValue": "tune_v2", | |
| "description": "The adapter tuning method. Available methods: tune,\ntune_v2.", | |
| "isOptional": true, | |
| "parameterType": "STRING" | |
| } | |
| } | |
| }, | |
| "outputDefinitions": { | |
| "parameters": { | |
| "endpoint_resource_name": { | |
| "parameterType": "STRING" | |
| }, | |
| "model_resource_name": { | |
| "parameterType": "STRING" | |
| } | |
| } | |
| } | |
| }, | |
| "schemaVersion": "2.1.0", | |
| "sdkVersion": "kfp-2.7.0" | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment