Goals: Add links that are reasonable and good explanations of how stuff works. No hype and no vendor content if possible. Practical first-hand accounts of models in prod eagerly sought.
![Screenshot 2023-12-18 at 10 40 27 PM](https://private-user-images.githubusercontent.com/3837836/291468646-4c30ad72-76ee-4939-a5fb-16b570d38cf2.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTUiLCJleHAiOjE3MTg5NzA3NjksIm5iZiI6MTcxODk3MDQ2OSwicGF0aCI6Ii8zODM3ODM2LzI5MTQ2ODY0Ni00YzMwYWQ3Mi03NmVlLTQ5MzktYTVmYi0xNmI1NzBkMzhjZjIucG5nP1gtQW16LUFsZ29yaXRobT1BV1M0LUhNQUMtU0hBMjU2JlgtQW16LUNyZWRlbnRpYWw9QUtJQVZDT0RZTFNBNTNQUUs0WkElMkYyMDI0MDYyMSUyRnVzLWVhc3QtMSUyRnMzJTJGYXdzNF9yZXF1ZXN0JlgtQW16LURhdGU9MjAyNDA2MjFUMTE0NzQ5WiZYLUFtei1FeHBpcmVzPTMwMCZYLUFtei1TaWduYXR1cmU9MzFiMjRmMmJkNzRhNWVlYmFiZTIzOGYyNTQ0ODNmMjQ2ZmE4NDA1YThkZjFkNDAyNzE5YjI1ODMzMDU3MjU2OSZYLUFtei1TaWduZWRIZWFkZXJzPWhvc3QmYWN0b3JfaWQ9MCZrZXlfaWQ9MCZyZXBvX2lkPTAifQ.Q16iLcME1kiOLI9F01sB58JuakoY3ExQmY_flf36dTU)
# NOTE: | |
# You can find an updated, more robust and feature-rich implementation | |
# in Zeno Build | |
# - Zeno Build: https://github.com/zeno-ml/zeno-build/ | |
# - Implementation: https://github.com/zeno-ml/zeno-build/blob/main/zeno_build/models/providers/openai_utils.py | |
import openai | |
import asyncio | |
from typing import Any |
import pytorch_lightning as pl | |
... | |
# override this method on pytorch-lightning model | |
def on_predict_epoch_end(self, results): | |
# gather all results onto each device | |
# find created world_size from pl.trainer | |
results = all_gather(results[0], WORLD_SIZE, self._device) | |
# concatenate on the cpu | |
results = torch.concat([x.cpu() for x in results], dim=1) | |
# output will not preserve input order. |
sudo apt-get remove --purge '^nvidia-.*' | |
sudo ubuntu-drivers autoinstall | |
sudo reboot |