yujiepan-work

import os

import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader, Dataset

import nncf
from nncf.config import NNCFConfig
from nncf.config.structures import (BNAdaptationInitArgs,

yujiepan-work

Joint magnitude pruning, quantization and distillation on BERT-base/SST-2

This model conducts unstructured magnitude pruning, quantization and distillation at the same time on BERT-base when finetuning on the GLUE SST2 dataset. It achieves the following results on the evaluation set:

• Torch accuracy: 0.9128
• OpenVINO IR accuracy: 0.9128
• Sparsity in transformer block linear layers: 0.80

Setup

yujiepan-work

from contextlib import contextmanager
from unittest.mock import patch

from optimum.intel.openvino import OVModelForQuestionAnswering
import pandas as pd

import datasets
import evaluate
from evaluate import evaluator
from transformers import AutoTokenizer, pipeline, AutoModelForQuestionAnswering

yujiepan-work

import os

os.environ['CUDA_VISIBLE_DEVICES'] = '-1'

from collections import defaultdict
import time
import numpy as np
import pandas as pd
from multiprocessing import Pool

yujiepan-work

import os
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'

from optimum.intel.openvino import OVModelForImageClassification
import pandas as pd
import numpy as np

import datasets
import evaluate  # Use pip install git+https://github.com/huggingface/evaluate.git
from evaluate import evaluator

yujiepan-work

yujiepan-work

# %%
import torch.utils.benchmark as benchmark
import torch.nn as nn
from dataclasses import dataclass
import torch
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '0'

class FeedForward(nn.Module):
    def __init__(self, input_dim, intermediate_size):

yujiepan-work

• OV_MODELS or TORCH_MODELS can be a model_id on huggingface or a local folder
• For text tasks, there is a patch to ensure the sequence length is fixed. Can be removed if the model accepts arbitrary shape

yujiepan-work

benchmark_app

yujiepan-work

import copy
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
from functools import partial
from pathlib import Path
import json
from dataclasses import dataclass
import typing

import torch