A context manager for matplotlib that will autoscale only to objects plotted within its context. Ref. http://stackoverflow.com/questions/7386872/make-matplotlib-autoscaling-ignore-some-of-the-plots

careful_autoscale.py

class careful_autoscale:
    def __init__(self, ax=None):
        self.ax = ax if ax else gca()
    
    def __enter__(self):
        self.xl = self.ax.get_xlim()
        self.yl = self.ax.get_ylim()
        self.lines = self.ax.get_lines()
        self.lines_visibility = [ l.get_visible() for l in self.lines ]
        [ l.set_visible(False) for l in self.lines ]
        self.ax.autoscale(True)

    def __exit__(self, type, value, traceback):
        self.ax.relim(visible_only=True)
        self.ax.autoscale_view()
        if len(self.lines) > 1:
            self.ax.set_xlim(
                min(self.ax.get_xlim()[0], self.xl[0]),
                max(self.ax.get_xlim()[1], self.xl[1]))
            self.ax.set_ylim(
                min(self.ax.get_ylim()[0], self.yl[0]),
                max(self.ax.get_ylim()[1], self.yl[1]))
        [ l.set_visible(v) for l,v in zip(self.lines, self.lines_visibility) ]
        
ax = axes()
ax.plot([0, 1, 2, 1, 3, 4], 'o', label="data")
ax.margins(0.1,0.1)
ax.plot([-5,10], [-1, 5], '-', label="bad fit", scaley=False, scalex=False)

with careful_autoscale(ax):
    ax.plot([-1, 0, -2, 2, 5], '^', label="more data")
    
with careful_autoscale(ax):
    ax.plot([0, 0, 0, 0, 0, 0, 0, 0], '*', label="even more data")
    
ax.legend(loc="best")