Prodigy hierarchical text classification

custom.js

function toggle(id) {
    var x = document.getElementById(id);
    if (id == "a"){
        reset("b")
    }else{
        reset("a")
    }
    if (x.style.display === "none") {
    x.style.display = "block";
    } else {
    x.style.display = "none";
    }
}

function reset(id){
    var x = document.getElementById(id);
    x.style.display = "none"
    var checkboxes = document.getElementsByClassName("checkbox");
    for(let elem in checkboxes){
        checkboxes[elem].checked = false;
    }
}

function update(){
    var checkboxes = document.getElementsByClassName("checkbox");

    var results = [];
    for(let elem in checkboxes){
        if(checkboxes[elem].checked){
            results.push(checkboxes[elem].id)
        }
    }
    prodigy.update({
        selected: results
    })
}

document.addEventListener('prodigyanswer', event => {
    reset("a")
    reset("b")
})

example.json

{
  "text": "Uber’s Lesson: Silicon Valley’s Start-Up Machine Needs Fixing",
  "meta": {
    "source": "The New York Times"
  },
  "_input_hash": 1886699658,
  "_task_hash": -1952856502,
  "_view_id": "blocks",
  "selected": [
    "sub-option a1",
    "sub-option a2"
  ],
  "answer": "accept",
  "_timestamp": 1688585735,
  "_annotator_id": "2023-07-05_15-35-30",
  "_session_id": "2023-07-05_15-35-30",
  "upper_selected": [
    "a"
  ]
}

labels.jsonl

{"a": ["sub-option a1","sub-option a2"], "b": ["sub-option b1","sub-option b2"]}

news_headlines.jsonl

{"text":"Uber\u2019s Lesson: Silicon Valley\u2019s Start-Up Machine Needs Fixing","meta":{"source":"The New York Times"}}
{"text":"Pearl Automation, Founded by Apple Veterans, Shuts Down","meta":{"source":"The New York Times"}}
{"text":"How Silicon Valley Pushed Coding Into American Classrooms","meta":{"source":"The New York Times"}}
{"text":"Women in Tech Speak Frankly on Culture of Harassment","meta":{"source":"The New York Times"}}
{"text":"Silicon Valley Investors Flexed Their Muscles in Uber Fight","meta":{"source":"The New York Times"}}
{"text":"Uber is a Creature of an Industry Struggling to Grow Up","meta":{"source":"The New York Times"}}
{"text":"\u2018The Internet Is Broken\u2019: @ev Is Trying to Salvage It","meta":{"source":"The New York Times"}}
{"text":"The South Park Commons Fills a Hole in the Tech Landscape","meta":{"source":"The New York Times"}}
{"text":"The Closing of the Republican Mind","meta":{"source":"The New York Times"}}
{"text":"Writers From the Right and Left on Trump Jr., the Future of the F.B.I., Health Care and More","meta":{"source":"The New York Times"}}
{"text":"Daily Report: From Lean to Fat Start-Ups","meta":{"source":"The New York Times"}}
{"text":"How Uber\u2019s Chief Is Gaining Even More Clout in the Company","meta":{"source":"The New York Times"}}
{"text":"As New Zealand Courts Tech Talent, Isolation Becomes a Draw","meta":{"source":"The New York Times"}}
{"text":"One Thing Silicon Valley Can\u2019t Seem to Fix","meta":{"source":"The New York Times"}}
{"text":"In Silicon Valley, a Voice of Caution Guides a High-Flying Uber","meta":{"source":"The New York Times"}}
{"text":"Silicon Valley Writes a Protest Letter Against Trump","meta":{"source":"The New York Times"}}
{"text":"Warriors, Tech\u2019s Team, Are Soaring Out of Reach","meta":{"source":"The New York Times"}}
{"text":"Silicon Valley\u2019s Most Elusive Beast","meta":{"source":"The New York Times"}}
{"text":"Wall Street and Silicon Valley Form an Uneasy Alliance","meta":{"source":"The New York Times"}}
{"text":"Tim O'Reilly Explains the Internet of Things","meta":{"source":"The New York Times"}}
{"text":"Alibaba I.P.O. May Unleash Global Fight Over Users","meta":{"source":"The New York Times"}}
{"text":"Disruptions: Looking Beyond Silicon Valley's Bubble","meta":{"source":"The New York Times"}}
{"text":"Twitter Outages Linked to Glitches and Site Upgrade","meta":{"source":"The New York Times"}}
{"text":"Out of the Loop in Silicon Valley","meta":{"source":"The New York Times"}}
{"text":"A Determined Outpost of Tiny Technology","meta":{"source":"The New York Times"}}
{"text":"In Silicon Valley, Recruiters Are Sending Out Their Own R\u00e9sum\u00e9s","meta":{"source":"The New York Times"}}
{"text":"Credit Crisis Spreads a Pall Over Silicon Valley","meta":{"source":"The New York Times"}}
{"text":"As Department Stores Close, Stitch Fix Expands Online","meta":{"source":"The New York Times"}}
{"text":"Tech Incubators on a Mission of Diversity","meta":{"source":"The New York Times"}}
{"text":"Meal-Delivery Start-Ups Take Aim at Your Dinner","meta":{"source":"The New York Times"}}
{"text":"If a Bubble Bursts in Palo Alto, Does It Make a Sound?","meta":{"source":"The New York Times"}}
{"text":"Web\u2019s Reach Binds N.S.A. and Silicon Valley Leaders","meta":{"source":"The New York Times"}}
{"text":"Daily Report: Other Nations Offer Visas to Compete With Silicon Valley","meta":{"source":"The New York Times"}}
{"text":"Bill Maher Concert Will Introduce New Yahoo! Comedy Channel","meta":{"source":"The New York Times"}}
{"text":"A Silicon Valley Dream Grows in Guatemala, Despite the Risks","meta":{"source":"The New York Times"}}
{"text":"One Country, Two Revolutions","meta":{"source":"The New York Times"}}
{"text":"The Class That Built Apps, and Fortunes","meta":{"source":"The New York Times"}}
{"text":"E-Ties That Bind","meta":{"source":"The New York Times"}}
{"text":"Tech Recruiting Clashes With Immigration Rules","meta":{"source":"The New York Times"}}
{"text":"On Day Care, Google Makes a Rare Fumble","meta":{"source":"The New York Times"}}
{"text":"Where Computer Artifacts Come Alive","meta":{"source":"The New York Times"}}
{"text":"Can Green Make Green?","meta":{"source":"The New York Times"}}
{"text":"In Silicon Valley, the Crash Seems Like Just Yesterday","meta":{"source":"The New York Times"}}
{"text":"\u2018I\u2019m Here to Help,\u2019 Trump Tells Tech Executives at Meeting","meta":{"source":"The New York Times"}}
{"text":"Peter Thiel\u2019s Bet on Donald Trump Pays Off","meta":{"source":"The New York Times"}}
{"text":"Peter Thiel Defends His Most Contrarian Move Yet: Supporting Trump","meta":{"source":"The New York Times"}}
{"text":"What It\u2019s Like to Fight Online Hate","meta":{"source":"The New York Times"}}
{"text":"Start-Ups Once Showered With Cash Now Have to Work for It","meta":{"source":"The New York Times"}}
{"text":"European Tech Scene Begins to Feel Silicon Valley\u2019s Woes","meta":{"source":"The New York Times"}}
{"text":"Toyota Invests $1 Billion in Artificial Intelligence in U.S.","meta":{"source":"The New York Times"}}
{"text":"Farhad and Mike\u2019s Week in Review: Twitter Problems","meta":{"source":"The New York Times"}}
{"text":"Steep Discounts a Boon for Customers, but a Gamble for Start-Ups","meta":{"source":"The New York Times"}}
{"text":"Apple's Diversity Mirrors Other Tech Companies'   ","meta":{"source":"The New York Times"}}
{"text":"Building a Better Battery","meta":{"source":"The New York Times"}}
{"text":"Disruptions: If It Looks Like a Bubble and Floats Like a Bubble ...","meta":{"source":"The New York Times"}}
{"text":"Disruptions: Even the Tech Elites Leave Gadgets Behind","meta":{"source":"The New York Times"}}
{"text":"Latest Product From Tech Firms: An Immigration Bill","meta":{"source":"The New York Times"}}
{"text":"Migrant Bill Seems to Fit Tech Sector Wish List","meta":{"source":"The New York Times"}}
{"text":"The Internet Gets Physical","meta":{"source":"The New York Times"}}
{"text":"The War Between Apple and Google Has Just Begun","meta":{"source":"The New York Times"}}
{"text":"Site Lets Investors See and Copy Experts\u2019 Trades","meta":{"source":"The New York Times"}}
{"text":"Amid Conference Halls and Keynote Speakers, a Rivalry Forms","meta":{"source":"The New York Times"}}
{"text":"Economy Has Become a Drag on Silicon Valley","meta":{"source":"The New York Times"}}
{"text":"Yahoo Decides to Release a Rosy Forecast","meta":{"source":"The New York Times"}}
{"text":"Silicon Valley Losing Middle-Wage Jobs","meta":{"source":"The New York Times"}}
{"text":"The Working-Class Millionaire","meta":{"source":"The New York Times"}}
{"text":"Andy Grove\u2019s Warning to Silicon Valley","meta":{"source":"The New York Times"}}
{"text":"Caution Rebuffed, Unicorns and Other Start-Ups Fixate on Rainbows","meta":{"source":"The New York Times"}}
{"text":"Kleiner Perkins, Disrupted","meta":{"source":"The New York Times"}}
{"text":"When Uber and Airbnb Meet the Real World","meta":{"source":"The New York Times"}}
{"text":"Yahoo Reveals Work Force Data, Joining Tech's Small Diversity Parade","meta":{"source":"The New York Times"}}
{"text":"Disruptions: The Echo Chamber of Silicon Valley","meta":{"source":"The New York Times"}}
{"text":"TimesCast Tech: Career and Family in Silicon Valley","meta":{"source":"The New York Times"}}
{"text":"Steve Jobs: He Brought the Show to Business","meta":{"source":"The New York Times"}}
{"text":"For Start-Ups, Late Bursts of Private Cash","meta":{"source":"The New York Times"}}
{"text":"AOL, Seeking to Revitalize Its Internet Role, Turns to a Candid Ex-Yahoo Executive","meta":{"source":"The New York Times"}}
{"text":"Buy.com Deal With eBay Angers Sellers","meta":{"source":"The New York Times"}}
{"text":"Silicon Valley Starts to Turn Its Face to the Sun","meta":{"source":"The New York Times"}}
{"text":"Silicon Valley\u2019s High-Tech Hunt for Colleague","meta":{"source":"The New York Times"}}
{"text":"Twitter\u2019s Troubles and Snap\u2019s Appeal: It\u2019s All About the Mojo","meta":{"source":"The New York Times"}}
{"text":"A Rarity at a Republican Convention: \u2018I Am Proud to Be Gay\u2019","meta":{"source":"The New York Times"}}
{"text":"Pinterest Hires Its First Head of Diversity","meta":{"source":"The New York Times"}}
{"text":"Daily Report: Narendra Modi, Indian Prime Minister, Conquers Silicon Valley","meta":{"source":"The New York Times"}}
{"text":"Inside Amazon: Reporter\u2019s Notebook","meta":{"source":"The New York Times"}}
{"text":"Hot Tech Start-Ups May Face a Long and Bumpy Fall","meta":{"source":"The New York Times"}}
{"text":"White House Takes Cybersecurity Pitch to Silicon Valley ","meta":{"source":"The New York Times"}}
{"text":"The PayPal Mafia\u2019s Golden Touch","meta":{"source":"The New York Times"}}
{"text":"What We're Reading","meta":{"source":"The New York Times"}}
{"text":"Will Boom Lead to Bust in Silicon Valley? ","meta":{"source":"The New York Times"}}
{"text":"Tech Industry Flexes Muscle in California Race","meta":{"source":"The New York Times"}}
{"text":"Disruptions: A Blogger Mocks the Denizens of Silicon Valley ","meta":{"source":"The New York Times"}}
{"text":"How Big Data Is Playing Recruiter for Specialized Workers","meta":{"source":"The New York Times"}}
{"text":"Daily Report: Multinationals Push Into Venture Capital","meta":{"source":"The New York Times"}}
{"text":"As Silicon Valley Cheers Yahoo Chief, Wall Street's Reaction Is Muted","meta":{"source":"The New York Times"}}
{"text":"Old Techies Never Die; They Just Can\u2019t Get Hired as an Industry Moves On","meta":{"source":"The New York Times"}}
{"text":"A Silicon Bubble Shows Signs of Reinflating","meta":{"source":"The New York Times"}}
{"text":"A Start-Up Says It Can Predict Others\u2019 Fate","meta":{"source":"The New York Times"}}
{"text":"More Than Games, a Net to Snare Social Networkers","meta":{"source":"The New York Times"}}
{"text":"Silicon Valley Shaped by Technology and Traffic","meta":{"source":"The New York Times"}}
{"text":"In Silicon Valley, Millionaires Who Don\u2019t Feel Rich","meta":{"source":"The New York Times"}}
{"text":"Lots of Froth but No Bubble","meta":{"source":"The New York Times"}}
{"text":"Start-Up Fervor Shifts to Energy in Silicon Valley","meta":{"source":"The New York Times"}}
{"text":"VeriSign Moves to Address an Internet Security Problem","meta":{"source":"The New York Times"}}
{"text":"China, Not Silicon Valley, Is Cutting Edge in Mobile Tech","meta":{"source":"The New York Times"}}
{"text":"Women in Tech Band Together to Track Diversity, After Hours","meta":{"source":"The New York Times"}}
{"text":"Facebook\u2019s Developer Conference Kicks Off","meta":{"source":"The New York Times"}}
{"text":"An Anonymous Satire of Silicon Valley Now Has a Publisher","meta":{"source":"The New York Times"}}
{"text":"Silicon Valley Bank Strengthens Its Roots","meta":{"source":"The New York Times"}}
{"text":"Daily Report: Rivalry Brewing Between Uber and Google","meta":{"source":"The New York Times"}}
{"text":"Behind the Cover Story: Yiren Lu on the Angst, Perks and Failures of the Tech Industry","meta":{"source":"The New York Times"}}
{"text":"Thorny Side Effects in Silicon Valley Tactic to Keep Control","meta":{"source":"The New York Times"}}
{"text":"Is New York's Tech Boom Sustainable?","meta":{"source":"The New York Times"}}
{"text":"Internet Giants Foster, and Threaten, Innovation Economy","meta":{"source":"The New York Times"}}
{"text":"Following Venture Capital for Signs of Tech to Come","meta":{"source":"The New York Times"}}
{"text":"At Social Site, Only the Businesslike Need Apply","meta":{"source":"The New York Times"}}
{"text":"When It Comes to Innovation, Geography Is Destiny","meta":{"source":"The New York Times"}}
{"text":"Tech Billionaire Backing Trump Suggests Silicon Valley Is Out of Touch","meta":{"source":"The New York Times"}}
{"text":"How Silicon Valley Treats a Trump Backer: Peter Thiel","meta":{"source":"The New York Times"}}
{"text":"Parents Ready for Some Love From Silicon Valley Companies","meta":{"source":"The New York Times"}}
{"text":"Scrutiny of Security Start-Ups May Signal Shift in Venture Funding","meta":{"source":"The New York Times"}}
{"text":"Marissa Mayer of Yahoo Says She’s Pregnant With Twins","meta":{"source":"The New York Times"}}
{"text":"A Novel Prompts a Conversation About How We Use Technology","meta":{"source":"The New York Times"}}
{"text":"Tech Industry Sets Its Sights on Gambling","meta":{"source":"The New York Times"}}
{"text":"Techies Break a Fashion Taboo","meta":{"source":"The New York Times"}}
{"text":"Back to the Future: The Netscape and Google I.P.O.'s","meta":{"source":"The New York Times"}}
{"text":"Where Are the Women Executives in Silicon Valley?","meta":{"source":"The New York Times"}}
{"text":"For Buyers of Web Start-Ups, Quest to Corral Young Talent","meta":{"source":"The New York Times"}}
{"text":"Digital Muse for Beat Poet","meta":{"source":"The New York Times"}}
{"text":"Amid Conference Halls and Keynote Speakers, a Rivalry Forms","meta":{"source":"The New York Times"}}
{"text":"Hostility Has Its Rewards","meta":{"source":"The New York Times"}}
{"text":"Going Public Caps Dream for a Maker of Software","meta":{"source":"The New York Times"}}
{"text":"Next Job for Obama? Silicon Valley Is Hiring","meta":{"source":"The New York Times"}}
{"text":"Airbnb and Others Set Terms for Employees to Cash Out","meta":{"source":"The New York Times"}}
{"text":"Google and Apple: the High-Tech Hippies of Silicon Valley","meta":{"source":"The New York Times"}}
{"text":"Silicon Valley, Seeking Diversity, Focuses on Blacks","meta":{"source":"The New York Times"}}
{"text":"As More Tech Start-Ups Stay Private, So Does the Money","meta":{"source":"The New York Times"}}
{"text":"Want a Steady Income? There\u2019s an App for That","meta":{"source":"The New York Times"}}
{"text":"Investing Early On for Insights, Not Profits","meta":{"source":"The New York Times"}}
{"text":"Angel Investors Lend Expertise as Well as Cash","meta":{"source":"The New York Times"}}
{"text":"Deal Makers Invade CES, the Land of Geeks","meta":{"source":"The New York Times"}}
{"text":"Techies Break a Fashion Taboo","meta":{"source":"The New York Times"}}
{"text":"Back to the Future: The Netscape and Google I.P.O.'s","meta":{"source":"The New York Times"}}
{"text":"Where Are the Women Executives in Silicon Valley?","meta":{"source":"The New York Times"}}
{"text":"For Buyers of Web Start-Ups, Quest to Corral Young Talent","meta":{"source":"The New York Times"}}
{"text":"Digital Muse for Beat Poet","meta":{"source":"The New York Times"}}
{"text":"Amid Conference Halls and Keynote Speakers, a Rivalry Forms","meta":{"source":"The New York Times"}}
{"text":"Hostility Has Its Rewards","meta":{"source":"The New York Times"}}
{"text":"Going Public Caps Dream for a Maker of Software","meta":{"source":"The New York Times"}}
{"text":"Next Job for Obama? Silicon Valley Is Hiring","meta":{"source":"The New York Times"}}
{"text":"Airbnb and Others Set Terms for Employees to Cash Out","meta":{"source":"The New York Times"}}
{"text":"In Surveillance Debate, White House Turns Its Focus to Silicon Valley","meta":{"source":"The New York Times"}}
{"text":"Silicon Valley\u2019s Youth Problem","meta":{"source":"The New York Times"}}
{"text":"Zuckerberg Remains the Undisputed Boss at Facebook","meta":{"source":"The New York Times"}}
{"text":"A Corporate Campus Made to Mirror Facebook","meta":{"source":"The New York Times"}}
{"text":"One on One: Tim Wu, Author of 'The Master Switch'","meta":{"source":"The New York Times"}}
{"text":"Spinning the Web: P.R. in Silicon Valley","meta":{"source":"The New York Times"}}
{"text":"To Survive, Net Start-Ups Slow Their Metabolism","meta":{"source":"The New York Times"}}
{"text":"Attending to the Needs of the Too-Busy","meta":{"source":"The New York Times"}}
{"text":"Silicon Valley Start-Ups Awash in Dollars, Again","meta":{"source":"The New York Times"}}
{"text":"New Social Sites Cater to People of a Certain Age","meta":{"source":"The New York Times"}}
{"text":"In Surveillance Debate, White House Turns Its Focus to Silicon Valley","meta":{"source":"The New York Times"}}
{"text":"Silicon Valley\u2019s Youth Problem","meta":{"source":"The New York Times"}}
{"text":"Zuckerberg Remains the Undisputed Boss at Facebook","meta":{"source":"The New York Times"}}
{"text":"A Corporate Campus Made to Mirror Facebook","meta":{"source":"The New York Times"}}
{"text":"One on One: Tim Wu, Author of 'The Master Switch'","meta":{"source":"The New York Times"}}
{"text":"Spinning the Web: P.R. in Silicon Valley","meta":{"source":"The New York Times"}}
{"text":"To Survive, Net Start-Ups Slow Their Metabolism","meta":{"source":"The New York Times"}}
{"text":"Attending to the Needs of the Too-Busy","meta":{"source":"The New York Times"}}
{"text":"Silicon Valley Start-Ups Awash in Dollars, Again","meta":{"source":"The New York Times"}}
{"text":"New Social Sites Cater to People of a Certain Age","meta":{"source":"The New York Times"}}
{"text":"Demand for Data Puts Engineers in Spotlight","meta":{"source":"The New York Times"}}
{"text":"Yahoo Sale Could Be Bad for Minnows","meta":{"source":"The New York Times"}}
{"text":"Silicon Valley Helped Create Trump, and That\u2019s Bad for It","meta":{"source":"The New York Times"}}
{"text":"Peter Thiel\u2019s Embrace of Trump Has Silicon Valley Squirming","meta":{"source":"The New York Times"}}
{"text":"In Silicon Valley, a Divide in Income and Politics","meta":{"source":"The New York Times"}}
{"text":"The Art of Failing Upward","meta":{"source":"The New York Times"}}
{"text":"Daily Report: The Tech News Cycle is About to Ramp Up","meta":{"source":"The New York Times"}}
{"text":"Netromancy","meta":{"source":"The New York Times"}}
{"text":"Unicorns Hunt for Talent Among Silicon Valley\u2019s Giants","meta":{"source":"The New York Times"}}
{"text":"Overvalued in Silicon Valley, but Don\u2019t Say \u2018Tech Bubble\u2019","meta":{"source":"The New York Times"}}
{"text":"Managers Turn to Computer Games, Aiming for More Efficient Employees","meta":{"source":"The New York Times"}}
{"text":"Stanford Women and Silicon Valley","meta":{"source":"The New York Times"}}
{"text":"What It Means to Be a \u2018Dad\u2019","meta":{"source":"The New York Times"}}
{"text":"What We're Reading","meta":{"source":"The New York Times"}}
{"text":"A Golden Age of Design","meta":{"source":"The New York Times"}}
{"text":"Court Rejects Deal on Hiring in Silicon Valley","meta":{"source":"The New York Times"}}
{"text":"What the Beats Deal Says About Apple: It Loves Tastemakers","meta":{"source":"The New York Times"}}
{"text":"Search for a Market Niche, and You Might Find a Crowd","meta":{"source":"The New York Times"}}
{"text":"Search for the 'Next Big Thing' Yields Soaring Valuations","meta":{"source":"The New York Times"}}
{"text":"Shifting Tech Scene Unsettles Big Players","meta":{"source":"The New York Times"}}
{"text":"Silicon Valley\u2019s Start-Up Machine","meta":{"source":"The New York Times"}}
{"text":"Opening a Gateway for Girls to Enter the Computer Field","meta":{"source":"The New York Times"}}
{"text":"Silicon Valley Says Step Away From the Device","meta":{"source":"The New York Times"}}
{"text":"Like the Apple Store Nearby, but This One Has Windows","meta":{"source":"The New York Times"}}
{"text":"A King of Connections Is Tech\u2019s Go-To Guy","meta":{"source":"The New York Times"}}
{"text":"One on One: Jaron Lanier","meta":{"source":"The New York Times"}}
{"text":"6 Months, $90,000 and (Maybe) a Great Idea","meta":{"source":"The New York Times"}}
{"text":"Microsoft\u2019s Failed Yahoo Bid Risks Online Growth","meta":{"source":"The New York Times"}}
{"text":"Why Old Technologies Are Still Kicking","meta":{"source":"The New York Times"}}
{"text":"Trying to Add a Pulse to a World of Machines","meta":{"source":"The New York Times"}}

recipe.py

import jinja2
from typing import Union
from pathlib import Path

import srsly
import prodigy
from prodigy.util import msg
from prodigy import set_hashes, get_stream


def load_template(path: Union[str, Path]) -> jinja2.Template:
    if not isinstance(path, Path):
        path = Path(path)
    if not path.suffix == ".jinja2":
        msg.fail(
            "Must supply jinja2 file.",
            exits=1,
        )
    with path.open("r", encoding="utf8") as file_:
        text = file_.read()
    return jinja2.Template(text, undefined=jinja2.DebugUndefined)

def get_upper(input_strings, dictionary):
    if input_strings is None: # if empty, return empty list
        return list()
    matching_keys = []
    for key, value in dictionary.items():
        if isinstance(value, list) and any(elem in value for elem in input_strings):
            matching_keys.append(key)
    return matching_keys

@prodigy.recipe(
        "textcat.hierarchical",
        dataset=("The dataset to save to", "positional", None, str),
        source=("The source data as a JSONL file", "positional", None, str),
        labels=("The label hierarchy as a JSONL file", "positional", None, str),
)
def textcat_hierarchical(
    dataset: str, 
    source: str,
    labels: str,
):

    options = list(srsly.read_jsonl(labels))[0]
    stream = get_stream(source, rehash=True, dedup=True)
    template = load_template("template.jinja2")

    def add_template(stream):
        for ex in stream:
            ex['html'] = template.render(options=options)
            yield set_hashes(ex)

    custom_js = Path("custom.js").read_text() 
    
    def before_db(examples):
        for ex in examples:
            del ex['html']
            ex['upper_selected'] = get_upper(ex.get('selected'), options))
        return examples

    blocks = [
        {"view_id": "text"},
        {"view_id": "html"},
    ]

    return {
        "view_id": "blocks",         
        "dataset": dataset,   # Name of dataset to save annotations       
        "stream": add_template(stream),  # Incoming stream of examples          
        "config": {
            "blocks": blocks,      
            "javascript": custom_js,
        },
        "before_db": before_db
    }

template.jinja2

<button onclick="toggle('a')">Option A</button>
<div id="a" style="display: none;">
    <form style="display: block;">
    {%- for reason in options["a"] -%}
        <input type="checkbox" class="checkbox" id="{{reason}}" name="{{reason}}" onchange="update()" style="margin: 0.4rem;"><label for="{{reason}}">{{reason}}</label><br>
    {%- endfor -%}
    </form>
</div>
<button onclick="toggle('b')">Option B</button>
<div id="b" style="display: none;">
<form style="display: block;">
    {%- for reason in options["b"] -%}
        <input type="checkbox" class="checkbox" id="{{reason}}" name="{{reason}}" onchange="update()" style="margin: 0.4rem;"><label for="{{reason}}">{{reason}}</label><br>
    {%- endfor -%}
    </form>
</div>

python3 -m prodigy textcat.hierarchical news-hier ./news_headlines.jsonl ./labels.jsonl -F recipe.py

textcat-hierarchical.mp4