Copy of Test-lighteval.ipynb

Eval-Arabic-LLMs-using-lighteval.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/alielfilali01/d486cfc962dca3ed4091b7c562a4377f/copy-of-test-lighteval.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RuUMMwfjLsu7"
      },
      "outputs": [],
      "source": [
        "# !apt-get install git-lfs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FN18Cy18Muht"
      },
      "outputs": [],
      "source": [
        "# Clone the repo\n",
        "!git clone https://github.com/huggingface/lighteval.git\n",
        "%cd lighteval\n",
        "\n",
        "# Install dependencies\n",
        "!pip install -e ."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7th6_GlkMuX_"
      },
      "outputs": [],
      "source": [
        "# Download the specific tasks file\n",
        "!wget https://raw.githubusercontent.com/huggingface/lighteval/main/tasks_examples/all_arabic_tasks.txt -O tasks_examples/all_arabic_tasks.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dJVFxUtMMuVo"
      },
      "outputs": [],
      "source": [
        "# Set Hugging Face authentication token if pushing results to the Hub\n",
        "import os\n",
        "os.environ[\"HUGGING_FACE_HUB_TOKEN\"] = \"YOUR_TOKEN_HERE\" # In case you care about pushing results/details to hub or eval gated models"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kc0IzAPUAqTJ"
      },
      "outputs": [],
      "source": [
        "# Run evaluations for openai-community/gpt2 model\n",
        "!python run_evals_accelerate.py --model_args=\"pretrained=openai-community/gpt2\" --custom_tasks community_tasks/arabic_evals.py --tasks tasks_examples/all_arabic_tasks.txt --override_batch_size 1 --save_details --output_dir=\"./output_gpt2\"\n",
        "\n",
        "# # Optional : Use --push_results_to_hub and --push_details_to_hub flags to push results to the Hub\n",
        "# !python run_evals_accelerate.py --model_args=\"pretrained=openai-community/gpt2\" --custom_tasks community_tasks/arabic_evals.py --tasks tasks_examples/all_arabic_tasks.txt --override_batch_size 1 --push_results_to_hub --results_org=\"YOUR_HF_USERNAME\" # Make sure to create a dataset named \"private_results\" under your profile\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "authorship_tag": "ABX9TyPGhoI9qQujrjIykDAeOhI8",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}