A way to overcome the difficulty of matplotib to deal with aspect ratio of images and their corresponding colorbars is to use ImageGrid. Here is an example to show how one can use ImageGrid to nicely add a vertical colorbar to an image.

colorbar_and_ImageGrid.py

#!/usr/bin/env python
# coding: utf-8

import matplotlib.pyplot as plt
import numpy as np
from mpl_toolkits.axes_grid1 import ImageGrid

"""
an example to show how one can use ImageGrid to nicely add a vertical colorbar to an image.
"""
# data to be plotted, width is 8 times larger than height for an aspect ratio of 1
data = np.arange(40*320).reshape((40, 320))

# use predefined style
plt.style.use("seaborn-paper")
# create a figure
F = plt.figure(1)
# clear the figure
F.clf()
# create an instace of ImageGrid
grid = ImageGrid(F, 111,
                     nrows_ncols=(1, 1),
                     direction="row",
                     axes_pad=0.05,
                     add_all=True,
                     label_mode="1",
                     share_all=True,
                     cbar_location="right",
                     cbar_mode="each",
                     cbar_size="5%",
                     cbar_pad="1%",
                     )
# plot the image and colorbar on ImageGrid
ax = grid[0]
# modify the aspect ratio of the image to 2 (one can try other values, 8 for instance for squared axises)
im = ax.imshow(np.arange(40*320).reshape((40, 320)), aspect=2)
cbar = grid.cbar_axes[0].colorbar(im)
cbar.set_label_text("scale")
ax.set_title("A nice title")
ax.set_ylabel("Z")
ax.set_xlabel("X")
plt.show()