FastEmbed

0_documents.json

{"url": "https://www.bbc.com/news/uk-politics-67296825", "title": "AI summit: Education will blunt AI risk to jobs, says Rishi Sunak - BBC News", "body": ["People should not be worried about the impact of AI on jobs because education reforms will boost skills, Rishi Sunak has said.", "Speaking after the UK's first AI safety summit, the prime minister said the technology would improve the economy in the long term.", "He added that new tools should be seen as a \"co-pilot\" to help people at work, rather than replacing them.", "The government's job should be to improve training, he told reporters.", "Mr Sunak said he recognised there was \"anxiety\" about the impact new AI tools could have on the workplace, but said it would enhance productivity over time. ", "\"We should look at AI much more as a co-pilot than something which is necessary going to replace someone's job. AI is a tool that can help almost anybody do their jobs better, faster, quicker.", "\"My job, the government's job, is to make sure we have a world-class education system,\" he added.", "\"That is my answer in a nutshell, that's why I don't want people to be worried, because we are building a world-class education system.\"", "Mr Sunak cited his recently-announced plan to introduce a new qualification for all school leavers in England, including some English and maths to 18.", "He also suggested efforts to improve technical training, and plans to boost adult education, would ensure that the UK could \"reap the benefits of AI economically\".", "His comments came at a press conference following a two-day summit on artificial intelligence safety attended by 28 countries, including the US and China, alongside tech bosses and academics.", "Trade unions, which have complained about not being represented at the event, have called for stronger measures to ensure jobs are protected as AI technology evolves. ", "At the summit, hosted at Bletchley Park in Buckinghamshire, several leading technology companies agreed to allow governments to safety-test their next generation of AI models before they are deployed.", "The voluntary document was signed by 10 countries and the EU, including the UK, US, Singapore and Canada. China was not a signatory.", "In a statement, the UK government said it would work with the Alan Turing Institute, a research body, to assess possible risks such as the potential for bias and misinformation.", "Mr Sunak said the testing regime would provide some \"independent assurance\" - adding that the firms developing new models cannot be expected to \"make their own homework\".", "His government has so far declined to announce legislation to regulate AI, arguing that existing regulators are best placed to mitigate the risks whilst the technology evolves.", "Mr Sunak told reporters that binding rules would \"likely be necessary,\" but stressed that the technology was still evolving and it was necessary to ensure it is done in \"the right way\".", "Before the summit, various unions and campaign groups warned the event would prove a \"missed opportunity\".", "In an open letter, they argued the event should have focused more on topics such as the impact of AI on employment law and smaller businesses, as well as policing and identity profiling.", "The summit has seen countries sign a declaration pledging more co-operation on research, to ensure the technology develops in a way that is \"human-centric, trustworthy and responsible\".", "Mr Sunak said he hoped the event would become the first in a series, with Korea and France also expressing a willingness to host further summits next year. ", "Some had criticised the inclusion of China at the event at a time of tense relations with West, despite the country being a key player in AI technology. ", "Mr Sunak defended the decision to invite the country, adding it \"wasn't an easy decision\" but that it was the \"right long-term decision\".", "\"Any serious conversation about AI safety has to engage the leading AI nations,\" he added. ", "On Wednesday, US Vice-President Kamala Harris announced the creation of the US AI Safety Institute, which the White House said would work alongside its UK counterpart. ", "Ms Harris had called for a focus on the \"everyday threats\" of AI, such as discrimination and disinformation, as well as \"existential\" fears.", "Earlier in the week, US President Joe Biden also signed an executive order, seeking to ensure \"America leads the way in seizing the promise and managing the risks of artificial intelligence\". ", "Some commentators had suggested the US's moves threatened to overshadow the UK's summit. ", "However, Mr Sunak welcomed the US executive order, calling it \"a deep and comprehensive demonstration of the potential of AI\"."]}
{"url": "https://www.bbc.com/news/technology-67285315", "title": "Rishi Sunak: AI firms cannot 'mark their own homework' - BBC News", "body": ["Monitoring the risks posed by artificial intelligence (AI) is too important to be left to big tech firms, Prime Minister Rishi Sunak has said. ", "He told the BBC that governments needed to take action and AI firms could not be left to \"mark their own homework\".", "He was speaking ahead of the AI Safety Summit, where a global declaration on managing AI risks has been announced.", "King Charles told delegates the issue required \"urgency, unity and collective strength\".", "It comes amid growing concerns about highly advanced forms of AI with as-yet unknown capabilities. ", "So far countries are only starting to address the potential risks, which may include breaches to privacy, cyberattacks and the displacement of jobs. ", "In an interview with the BBC at Downing Street, Mr Sunak AI was a \"transformative technology\" that could have huge benefits in the NHS or in schools.", "But he said he wanted the UK and other countries to be able \"do the testing that is necessary to make sure that we are keeping our citizens and everyone at home safe\".", "\"There has to be governments or external people who do that work,\" he said.", "Speaking to the BBC's technology editor Zoe Kleinman, he said that many AI firms had already given the UK access to their models before their release.", "And he claimed the UK was \"investing more\" AI risk management than any other country.", "\"We've already invested £100 million in our task force, which will become our Safety Institute,\" he said. ", "\"And we're attracting the best and the brightest researchers from around the world to come and work in that institution.\"", "Around 100 world leaders, tech bosses and academics are currently gathering at the UK's first AI safety summit at Bletchley Park, in Buckinghamshire.", "Earlier on Wednesday, the delegates agreed the world's first ever \"international statement\" on so called frontier AI - the government's term for AI that could exceed the capabilities of today's most advanced systems.", "The Bletchley Declaration calls for global cooperation on tackling the risks, which include potential breaches to privacy and the displacement of jobs. ", "Signed by 28 countries and the EU, it also says AI should be kept \"safe, in such a way as to be human-centric, trustworthy and responsible\".", "Dr Caitlin Bentley, AI education lecturer at King's College London, said the declaration  was an \"important milestone\" in promoting the \"responsible AI development\".", "However, she said more investment in AI education was needed to ensure \"AI is not only responsible, but equitable in its effects\" with the benefits felt by all.", "In his BBC interview, the prime minister defended a planned discussion with controversial tech billionaire Elon Musk on Thursday night, saying he could bring \"something valuable to the conversation\".", "\"Elon Musk for a long time has both been an investor and developer of AI technologies himself,\" said Mr Sunak. ", "\"For over a decade, he's been also talking about the potential risks that they pose and the need for countries and companies to work together to manage and mitigate against those risks.\"", "Mr Musk arrived at the summit on Wednesday morning, having warned the day before that AI could lead to the extinction of humanity.", "But many experts consider warnings like this overblown. ", "\"We've got representatives from all the major AI companies here at the summit,\" said Mr Sunak. ", "\"And that's crucial, because countries will need to work together with the companies that are developing the technology.\"", "Those appearing at the summit are discussing how best to maximise the benefits of AI - such as discovering new medicines and tackling climate change - while minimising the risks.", "The summit's priorities include the threat of bio-terrorism and cyber attacks.", "Speaking ahead of the event in London, US Vice President Kamala Harris said that world leaders \"must address the full spectrum of AI risks to humanity\" and listed examples of faulty algorithms in healthcare, the use of AI in making \"deepfakes\", misinformation and biased facial recognition. ", "China has also backed international cooperation on AI, with the country's Vice Minister for Science and Technology, Wu Zhaohui, calling for \"global collaboration to share knowledge and make AI technologies available to the public\"."]}
{"url": "https://www.bbc.com/news/uk-67302048", "title": "Killer robots and work-free lives - Sunak quizzes Musk on AI future - BBC News", "body": ["Deadly robots that can climb trees, AI friends and a work-less future were among the topics as Rishi Sunak sat down with Elon Musk.", "The prime minister held a highly unusual \"in conversation\" event with the billionaire X and Tesla owner at the end of this week's summit on artificial intelligence.", "Throughout the wide-ranging and chummy discussion, Mr Musk held court as the prime minister asked most of the questions.", "The pair talked about how London was a leading hub for the AI industry and how the technology could transform learning.", "But the chat took some darker turns too, with Mr Sunak recognising the \"anxiety\" people have about jobs being replaced, and the pair agreeing on the need for a \"referee\" to keep an eye on the super-computers of the future.", "Tech investor and inventor Mr Musk has put money into AI firms and has employed the technology in his driverless Tesla cars - but he's also on the record about his fears it could threaten society and human existence itself.", "\"There is a safety concern, especially with humanoid robots - at least a car can't chase you into a building or up a tree,\" he told the audience.", "Mr Sunak - who is keen to see investment in the UK's growing tech industry - replied: \"You're not selling this.\"", "It's not every day you see the prime minister of a country interviewing a businessman like this, but Mr Sunak seemed happy to play host to his famous guest.", "And if he seemed like he was enjoying it, it should be no surprise - he previously lived in California, home to Silicon Valley, and his love of all things tech is well-documented.", "In a hall that size, Mr Musk was difficult to hear and mumbled through his elaborate musings about the future, but refrained from any off-the-cuff remarks that might have caused Downing Street embarrassment. ", "The event was held in front of invited guests from the tech industry in a lavish hall in central London's Lancaster House.", "Unusually for an event involving the prime minister, TV cameras were not allowed inside, with Downing Street instead releasing their own footage. ", "Some reporters were allowed to observe - but told they could not ask questions. ", "The pair discussed the potential benefits of AI, with Mr Musk saying: \"One of my sons has trouble making friends and an AI friend would be great for him.\"", "There was also agreement on the possibilities AI presents for young people's learning, with Mr Musk saying it could be \"the best and most patient tutor\".", "But there was a stark warning on the potentially ruinous impact it could have on traditional jobs. ", "\"We are seeing the most destructive force in history here,\" Mr Musk said, before speculating: \"There will come a point where no job is needed - you can have a job if you want one for personal satisfaction but AI will do everything.", "\"It's both good and bad - one of the challenges in the future will be how do we find meaning in life.\"", "Amid all the philosophising, there was little in the way of new announcements about how the technology will be employed and regulated in the UK - aside from the prime minister's promise that AI could be used to improve the government's own website.", "Mr Musk was one of the star guests at this week's summit - but it briefly looked like the event with Mr Sunak might be a little overshadowed.", "Hours before it was due to begin, Mr Musk took to his own website X, formerly known as Twitter, to take a swipe at the summit.", "As Mr Sunak was on his feet giving his final press conference at Bletchley Park, Mr Musk shared a cartoon parodying an \"AI Safety Summit\".", "It depicted caricatures representing the UK, European Union, China and the US with speech bubbles reading \"We declare that AI posses a potentially catastrophic risk to humankind\" - while their thought bubbles read \"And I can not wait to develop it\".", "But in the end, the pair appeared at ease together, and Mr Sunak in particular looked in his element - perhaps even slightly bowled over by the controversial billionaire, who he called a \"brilliant innovator and technologist\".", "From the cheap seats behind the dignitaries of the tech world, it was hard to put your finger on who was really the powerful one out of this pair. ", "Was it Mr Sunak as he asked the celeb tech billionaire questions? Or was it Mr Musk, who did much of the talking?", "Either way, both men hope to have a say in whatever our AI future has in store for us.", "Additional reporting by Tom Gerken"]}

1_start.py

import jsonlines
with jsonlines.open("data/documents.json") as documents_file:
    documents = [row for row in documents_file]

documents[0].keys()

sentence = documents[0]["body"][0]
sentence

from fastembed.embedding import FlagEmbedding as Embedding

fe_model = Embedding(model_name="BAAI/bge-base-en-v1.5")

fe_model.embed(sentence)

list(fe_model.embed(sentence))
list(fe_model.embed(sentence))[0][:10]


list(fe_model.embed(f"passage: {sentence}"))[0][:10]
list(fe_model.passage_embed(sentence))[0][:10]

list(fe_model.embed(f"query: {sentence}"))[0][:10]
list(fe_model.query_embed(sentence))[0][:10]

2_chroma.py

import chromadb

chroma_client = chromadb.Client()
collection = chroma_client.create_collection("ai_articles")

for document in documents:        
    url = document["url"]
    title = document["title"]
    passages = document["body"]
    
    embeddings = [e.tolist() for e in fe_model.passage_embed(passages)]
    
    collection.add(
        documents = passages,
        embeddings = embeddings,
        ids = [f"{url}_{id}" for id in range(0, len(passages))],
        metadatas =[{
            "url": url, 
            "title": title
        } for _ in range(0, len(passages))] 
    )
    
query = "query: Is there going to be regulation of AI?"
results = collection.query(
    query_embeddings=[e.tolist() for e in fe_model.embed(query)],
    n_results=3,
)

query = "query: Where was the summit held?"
results = collection.query(
    query_embeddings=[e.tolist() for e in fe_model.embed(query)],
    n_results=3,
)