just my idea of what it should look like

multiple_pipelines.yml

tests:
  common:
    source_files: "submitted_files/*.{c,pas,cs}"
    additional_header: "additional_files/customlib.h"
  cases:
    - test_1:
      pipeline: "recodex/compile_exec_eval"
      test_input: "my_test_input_1"
      test_expected_output: "my_test_output_1"
    - test_2:
      pipeline: "my_compile_exec_eval"
      test_input: "my_test_input_2"
      test_expected_output: "my_test_output_2"
    - test_abc:
      pipeline: "my_compile_exec_eval"
      test_input: "my_test_input_3"
      test_expected_output: "my_test_output_3"
    - test_456:
      pipeline: "other_pipeline"
      test_input: "my_test_input_4"
      test_expected_output: "my_test_output_4"
pipelines:
  my_compile_exec_eval:
    input:
      - "source_code"
      - "test_input"
      - "test_expected_output"
    output:
      score: "my_judge_score"
    tasks:
      my_compilation:
        type: "compile"
        input:
          sources:
            - "...source_files"
            - "additional_header"
        output:
          binary: "my_executable_binary"
      my_execution:
        type: "execute"
        input:
          binary: "my_executable"
          stdin: "test_input"
        output:
          stdout: "my_output"
      my_judge:
        type: "judge"
        input:
          expected: "test_expected_output"
          actual: "my_output"
        output:
          score: "my_judge_score"
  other_pipeline:
    ...

simple_compile-exec-eval.yaml

recodex:
  input:
    src_file:
      name:
        - "*.cs"
        - "*.pas"
        - "*.c"
      type: "file[]"
    test_input:
      name: "*.in"
      type: "file[]"
    test_output:
      name: "*.out"
      type: "file[]"
  output:
    score: "judge_score"

compile:
  block_type: "compilation"
  options:
    language: "detect"
  input:
    sources: "src_file"
  output:
    binary: "executable_a"

exec:
  block_type: "run_binary"
  input:
    binary: "executable_a"
    stdin: "test_input[]"
  output:
    stdout: "actual_output"

eval:
  block_type: "judge"
  options:
    type: "strict"
  input:
    expected: "test_output[]"
    actual: "actual_output"
  output:
    score: "judge_score"