YAML Validator for Codalab Competitions v2

data.yaml

title: 'Hello world!'
image: 'some_image.png'

pages:
  - title: 'Welcome!'
    path: 'welcome.md'
  - title: 'How to compete'
    path: 'how_to.md'

phases:
  - name: 'First phase, specify many tasks!'
    description: 'A description'
    index: 0
    color: 'white'
    max_submissions: 100
    max_submissions_per_day: 1
    execution_time_limit_ms: 100
    start: 2018-11-06 22:16:16
    end: 2018-11-07 22:16:16

    files:
        - name: 'Public Data'
          description: 'Some public data'
          path: 'public_data.zip'
        - name: 'Starting Kit'
          description: 'Some starting kit'
          path: 'starting_kit.zip'

    tasks:
      - name: "Test task"
        description: "A test task"
        ingestion_module:
          name: "Test ingestion module"
          description: "A test ingestion module"
          ingestion_program: 'ingestion_program.zip'
          input_data: 'input_data.zip'
          only_during_scoring: True
        scoring_module:
          name: "Test solution"
          description: "A test solution"
          scoring_program: 'scoring_program.zip'
          reference_data: 'reference_data.zip'
    solutions:
      - name: 'Solution!'
        description: 'A very interesting solution'
        path: 'solution.zip'

leaderboards:
  - title: 'Leaderboard'
    key: 'main_leaderboard'
    index: 0
    #force_best_submission_to_leaderboard: true
    columns:
      - title: 'Score'
        key: 'score'
        index: 0
        #computation: enum('avg', required=False)
        #computation_indexes: str(required=False)
        sorting: 'asc'
        decimal_count: 0

schema.yaml

title: str(min=8, max=255)
image: str(max=1024)
pages: list(include('page'))
phases: list(include('phase'))
leaderboards: list(include('leaderboard'))

---

page:
  title: str(max=32)
  path: str(max=1024)

phase:
  name: str(max=128)
  description: str(max=1024)
  index: int(max=99)
  color: enum('white', 'orange', 'yellow', 'green', 'blue', 'purple')
  max_submissions: int(required=False)
  max_submissions_per_day: int(required=False)
  execution_time_limit_ms: int(max=5184000, required=False)
  start: timestamp(required=False)
  end: timestamp(required=False)

  files: list(include('file'))

  tasks: list(include('task'))
  solutions: list(include('solution'))

file:
  name: str(max=128)
  description: str(max=1024)
  path: str(max=1024)

task:
  name: str(max=128)
  description: str(max=1024)
  ingestion_module: include('ingestion_module')
  scoring_module: include('scoring_module')

ingestion_module:
  name: str(max=128, required=False)
  description: str(max=1024, required=False)
  ingestion_program: str(max=1024)
  input_data: str(max=1024, required=False)
  only_during_scoring: bool(required=False)

scoring_module:
  name: str(max=128, required=False)
  description: str(max=1024, required=False)
  scoring_program: str(max=1024)
  reference_data: str(max=1024)

solution:
  name: str(max=128)
  description: str(max=1024)
  path: str(max=1024)

leaderboard:
  title: str(max=128)
  key: str(max=128)
  index: int()
  columns: list(include('leaderboard_column'))
  force_submission_to_leaderboard: bool(required=False)
  force_best_submission_to_leaderboard: bool(required=False)
  disallow_leaderboard_modifying: bool(required=False)

leaderboard_column:
  title: str(max=128)
  key: str(max=128)
  index: int()
  computation: enum('avg', required=False)
  computation_indexes: str(required=False)
  sorting: enum('asc', 'desc')
  decimal_count: int(required=False)

yaml_validator.py

import yamale


schema = yamale.make_schema('./schema.yaml')

# Create a Data object
data = yamale.make_data('./data.yaml')

# Validate data against the schema. Throws a ValueError if data is invalid.
result = yamale.validate(schema, data)

if not result:
    raise Exception("Empty input")

print("Valid yaml!")