Software taught on physics - Code and cleaned data used.

README.md

This gist includes some code using pandas to visualise the data from the results in the survey and the "cleaned" version of such dataset (partly done with OpenRefine - lost script - and some bits were done manually).

analysis.py

import itertools
from cycler import cycler

import pandas as pd
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import cartopy.crs as ccrs
import cartopy.io.shapereader as shpreader




plt.style.use('fivethirtyeight')
morecolors = ['#348ABD', '#A60628', '#7A68A6', '#467821', '#D55E00', '#CC79A7', '#56B4E9', '#009E73', '#F0E442', '#0072B2']
plt.rcParams['axes.prop_cycle'] = plt.rcParams['axes.prop_cycle'].concat(cycler('color', reversed(morecolors)))
all_colors = plt.rcParams['axes.prop_cycle'].by_key()['color']

table = pd.read_csv('TPAResponseClean.csv')
table['Year'] = pd.to_datetime(table['Year'], format='%Y')

# Countries covered
countries = table.groupby('Country')['University'].count()

shapename = 'admin_0_countries'
countries_shp = shpreader.natural_earth(resolution='110m',
                                        category='cultural', name=shapename)

cmap = mpl.cm.viridis
fig = plt.figure(figsize=(10, 7))
ax = plt.axes(projection=ccrs.Robinson())
ax1 = fig.add_axes([0.1, 0.90, 0.83, 0.05])
for country in shpreader.Reader(countries_shp).records():
    name = country.attributes['NAME_LONG']
    if name in countries:
        ax.add_geometries(country.geometry, ccrs.PlateCarree(),
                          facecolor=cmap(countries[name] / countries.max(), 1),
                          label=name)
    else:
        ax.add_geometries(country.geometry, ccrs.PlateCarree(),
                          facecolor='#F0F0F0',
                          label=name)
norm = mpl.colors.Normalize(vmin=0, vmax=countries.max())
cb1 = mpl.colorbar.ColorbarBase(ax1, cmap=cmap,
                                norm=norm,
                                orientation='horizontal')

fig.savefig('participation_country.png')



# Years covered
fig, ax = plt.subplots()
table.groupby('Year').count()['Country'].resample('Y').mean().fillna(0).plot(ax=ax)
fig.tight_layout()
fig.savefig('people_answer.png')



# Define languages to group
proclangs = ['Fortran', 'C', 'Pascal']
oolangs = ['C++', 'Java']
interplangs = ['IDL', 'Matlab', 'Python']
tools = ['Excel', 'IRAF']
langs = proclangs + oolangs + interplangs + tools


# programming / #year vs year
fig, ax = plt.subplots(figsize=(10, 7))
table.groupby('Year').count()[langs].div(table.groupby('Year').count()['Country'], axis=0).resample('Y').mean().fillna(0).plot(ax=ax)
ax.set_ylim(0, 1.1)
plt.legend(langs)
fig.tight_layout()
fig.savefig('lang_year_all.png')


count = 0
for name, langs_i in zip(['procedural', 'oo', 'interp', 'tools'], [proclangs, oolangs, interplangs, tools]):
    fig, ax = plt.subplots(figsize=(10, 7))
    ax.set_prop_cycle(cycler('color', all_colors[count:]))
    ax.set_ylim(0, 1.1)
    table.groupby('Year').count()[langs_i].div(table.groupby('Year').count()['Country'], axis=0).resample('Y').mean().fillna(0).plot(ax=ax)
    plt.legend(langs_i)
    fig.tight_layout()
    fig.savefig('lang_year{}.png'.format(name))
    count += len(langs_i)


programming = proclangs + oolangs + interplangs
langs_hours = [n + ' (Hours)' for n in langs]
table['sumhours'] = table[langs_hours].sum(axis=1).fillna(0)
programming_hours = [n + ' (Hours)' for n in programming]
table['progsumhours'] = table[programming_hours].sum(axis=1).fillna(0)
programming_class = [n + ' (Class)' for n in  programming]


fig, ax = plt.subplots(figsize=(10, 7))
table.sort_values('sumhours')[langs_hours].plot.barh(stacked=True, ax=ax, width=1)
ax.get_yaxis().set_visible(False)
ax.set_xlabel('Hours')
plt.legend(langs)
fig.savefig('allhours.png')
print(' Number of entries with no 0 programming {}'.format(table['sumhours'].where(table['sumhours'] > 0).count()))


#Remove tools
fig, ax = plt.subplots(figsize=(10, 7))
table.sort_values('sumhours')[programming_hours].plot.barh(stacked=True, ax=ax, width=1)
ax.get_yaxis().set_visible(False)
ax.set_xlabel('Hours')
plt.legend(programming)
fig.savefig('allhours_programming.png')


# Without numerical methods
def no_opt(x):
    langs = ['Fortran', 'IDL', 'C', 'C++', 'Matlab', 'Python', 'Java', 'Pascal']
    programming_hours = [n + ' (Hours)' for n in langs]
    programming_class = [n + ' (Class)' for n in langs]
    total = 0
    for hour,subj in zip(programming_hours, programming_class):
        if isinstance(x[subj], str):
            if 'CS' not in x[subj] and 'Computer Science' not in x[subj] and 'COMP' not in x[subj]:
                if 'optional' not in x[subj].lower():
                    if 'summer' not in x[subj].lower():
                        total += x[hour]
    return total

fig, ax = plt.subplots(figsize=(10, 7))
table['progsum_nonumeric'] = table.apply(no_opt, axis=1)
table.sort_values('sumhours')['progsum_nonumeric'].plot.barh(stacked=True, ax=ax, width=1, color=all_colors[0])
ax.get_yaxis().set_visible(False)
ax.set_xlabel('Hours')
fig.savefig('allhours_programming_noopt.png')


def no_opt_numeric(x):
    """
    function to be applied to a pandas dataframe. It adds hours across the rows.
    """
    langs = ['Fortran', 'IDL', 'C', 'C++', 'Matlab', 'Python', 'Java', 'Pascal']
    programming_hours = [n + ' (Hours)' for n in langs]
    programming_class = [n + ' (Class)' for n in langs]
    total = 0
    for hour,subj in zip(programming_hours, programming_class):
        if isinstance(x[subj], str) and  'numeric' not in x[subj].lower():
            if 'CS' not in x[subj] and 'Computer Science' not in x[subj] and 'COMP' not in x[subj]:
                if 'optional' not in x[subj].lower():
                    if 'summer' not in x[subj].lower():
                        total += x[hour]
    return total

fig, ax = plt.subplots(figsize=(10, 7))
table['progsum_noextra'] = table.apply(no_opt_numeric, axis=1).fillna(0)
table.sort_values('sumhours')['progsum_noextra'].plot.barh(stacked=True, ax=ax, width=1, color=all_colors[0])
ax.get_yaxis().set_visible(False)
ax.set_xlabel('Hours')
fig.savefig('allhours_programming_nonum.png')


fig, ax = plt.subplots(figsize=(10, 7))
table.sort_values('progsum_noextra')['progsum_noextra'].plot.barh(stacked=True, ax=ax, width=1, color=all_colors[0])
ax.get_yaxis().set_visible(False)
ax.set_xlabel('Hours')
fig.savefig('allhours_programming_nonum_sorted.png')


print(' Number of entries with no 0 programming {}'.format(table['progsum_noextra'].where(table['progsum_noextra'] > 0).count()))

fig, ax = plt.subplots(figsize=(10, 7))
table.groupby('Year')['progsum_noextra'].sum().div(table.groupby('Year').count()['Country'], axis=0).resample('Y').mean().fillna(0).plot(ax=ax)
fig.tight_layout()
fig.savefig('lang_year_noextra.png')



# For how long F77 has been taught?
years77 = []
years95 = []
for y,f in table[['Year', 'Fortran']].values:
    if not f is np.nan and '77' in f:
        years77.append(y)
        if y >= y95:
            years95.append(y)
print('{} for F77, {} after 95. Latest on {:%Y}'.format(len(years77), len(years95), sorted(years77)[-1]))

TPAResponseClean.csv