json schema for pachyderm pipeline spec

README.md

JSON Schema

to use in visuat studio code:

in .vscode/settings.json

"json.schemas": [
    {
      "fileMatch": [
        "deploy/*.json"
      ],
      "url": "https://gist.githubusercontent.com/casualjim/3fe382db0ccce8a9ca64b862f7773614/raw/a1a11449399d532360d2ba0c5280723b671d7607/patchydermspec.json"
    }
  ]

Then create your pipeline specifications in: deploy/*.json and you get some help from the editor

patchydermspec.json

{
  "$schema": "http://json-schema.org/draft-04/schema",
  "title": "Pipeline specification",
  "type": "object",
  "required": [
    "pipeline",
    "transform"
  ],
  "properties": {
    "pipeline": {
      "type": "object",
      "required": [
        "name"
      ],
      "properties": {
        "name": {
          "type": "string"
        }
      }
    },
    "description": {
      "type": "string"
    },
    "transform": {
      "type": "object",
      "properties": {
        "image": {
          "type": "string"
        },
        "cmd": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "stdin": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "env": {
          "type": "object",
          "additionalProperties": {
            "type": "string"
          }
        },
        "secrets": {
          "type": "array",
          "items": {
            "type": "object",
            "properties": {
              "name": {
                "type": "string"
              },
              "mount_path": {
                "type": "string"
              }
            }
          }
        },
        "image_pull_secrets": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "accept_return_code": {
          "type": "array",
          "items": {
            "type": "integer",
            "format": "int64"
          }
        },
        "debug": {
          "type": "boolean"
        },
        "user": {
          "type": "string"
        },
        "working_dir": {
          "type": "string"
        }
      }
    },
    "parallelism_spec": {
      "oneOf": [{
          "type": "object",
          "properties": {
            "constant": {
              "type": "integer"
            }
          },
          "additionalProperties": false
        },
        {
          "type": "object",
          "properties": {
            "coefficient": {
              "type": "number"
            }
          },
          "additionalProperties": false
        }
      ]
    },
    "resource_requests": {
      "type": "object",
      "properties": {
        "memory": {
          "type": "string"
        },
        "cpu": {
          "type": "number"
        }
      }
    },
    "resource_limits": {
      "type": "object",
      "properties": {
        "memory": {
          "type": "string"
        },
        "cpu": {
          "type": "number"
        },
        "gpu": {
          "type": "number"
        }
      }
    },
    "datum_timeout": {
      "type": "string"
    },
    "job_timeout": {
      "type": "string"
    },
    "input": {
      "anyOf": [
        {
          "type": "object",
          "properties": {
            "atom": {
              "$ref": "#/definitions/atom"
            }
          }
        },
        {
          "type": "object",
          "properties": {
            "cron": {
              "$ref": "#/definitions/cron"
            }
          }
        },
        {
          "type": "object",
          "properties": {
            "git": {
              "$ref": "#/definitions/git"
            }
          }
        },
        {
          "type": "object",
          "properties": {
            "union": {
              "$ref": "#/definitions/union"
            }
          }
        },
        {
          "type": "object",
          "properties": {
            "cross": {
              "$ref": "#/definitions/cross"
            }
          }
        }
      ]
    },
    "output_branch": {
      "type": "string"
    },
    "egress": {
      "type": "object",
      "properties": {
        "URL": {
          "type": "string",
          "format": "uri"
        }
      }
    },
    "standby": {
      "type": "boolean"
    },
    "incremental": {
      "type": "boolean"
    },
    "cache_size": {
      "type": "string"
    },
    "enable_stats": {
      "type": "boolean"
    },
    "service": {
      "type": "object",
      "properties": {
        "internal_port": {
          "type": "integer"
        },
        "external_port": {
          "type": "integer"
        }
      }
    },
    "max_queue_size": {
      "type": "integer"
    },
    "chunk_spec": {
      "type": "object",
      "properties": {
        "number": {
          "type": "integer"
        },
        "size_bytes": {
          "type": "integer"
        }
      }
    }
  },
  "definitions": {
    "atom": {
      "type": "object",
      "properties": {
        "name": {
          "type": "string"
        },
        "repo": {
          "type": "string"
        },
        "branch": {
          "type": "string"
        },
        "glob": {
          "type": "string"
        },
        "lazy": {
          "type": "boolean"
        },
        "empty_files": {
          "type": "boolean"
        }
      }
    },
    "cron": {
      "type": "object",
      "properties": {
        "name": {
          "type": "string"
        },
        "spec": {
          "type": "string"
        },
        "repo": {
          "type": "string"
        },
        "start": {
          "type": "string",
          "format": "date-time"
        }
      }
    },
    "git": {
      "type": "object",
      "properties": {
        "name": {
          "type": "string"
        },
        "branch": {
          "type": "string"
        },
        "URL": {
          "type": "string",
          "format": "uri"
        }
      }
    },
    "cross": {
      "type": "array",
      "items": {
        "anyOf": [
          {
            "type": "object",
            "properties": {
              "atom": {
                "$ref": "#/definitions/atom"
              }
            }
          },
          {
            "type": "object",
            "properties": {
              "union": {
                "$ref": "#/definitions/union"
              }
            }
          }
        ]
      }
    },
    "union": {
      "type": "array",
      "items": {
        "anyOf": [
          {
            "type": "object",
            "properties": {
              "atom": {
                "$ref": "#/definitions/atom"
              }
            }
          },
          {
            "type": "object",
            "properties": {
              "cross": {
                "$ref": "#/definitions/cross"
              }
            }
          }
        ]
      }
    }
  }
}