promptfooconfig-filter-second-run.yml

# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
description: "Custom Provider"

prompts:
  - "say {{item}}"

providers:
  - id: file://./provider.js

tests:
  - vars:
      item: bananas
    assert:
      - type: contains
        value: "banana"
  - vars:
      item: apples
    assert:
      - type: contains
        value: "apple"
    provider:
      id: file://./provider2.js

promptfooconfig-first-run.yml

# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
description: "Custom Provider"

prompts:
  - "say {{item}}"

providers:
  - id: file://./provider.js

tests:
  - vars:
      item: bananas
    assert:
      - type: contains
        value: "banana"
  - vars:
      item: apples
    assert:
      - type: contains
        value: "apple"
    provider:
      id: file://./provider.js

provider.js

export default class BananaProvider {
  providerId = "banana";

  constructor(options) {
    this.providerId = options.id || "langchain";
  }

  id() {
    return this.providerId;
  }

  callApi = async () => {
    return {
      output: "banana",
    };
  };
}

provider2.js

export default class BananaProvider {
  providerId = "banana";

  constructor(options) {
    this.providerId = options.id || "langchain";
  }

  id() {
    return this.providerId;
  }

  callApi = async () => {
    return {
      output: "banana",
    };
  };
}