anderser

from django.conf.urls.defaults import *
from timelineit.views import *
from django.views.decorators.cache import cache_control,never_cache

urlpatterns = patterns('timelineit.views',
    
    url(r'^desk/(?P<slug>[-\w\d]+)', 
        never_cache(timeline_preview), 
        name="timeline_preview"),
                       

anderser

class CachedModelResource(ModelResource):

    def create_response(self, *args, **kwargs):
        resp = super(CachedModelResource, self).create_response(*args, **kwargs)
        resp['Cache-Control'] = "max-age=600"
        return resp

anderser

#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
import shlex, subprocess
import logging
from django.conf import settings

logger = logging.getLogger('pagetracker.grabber')

def grab_page(url, outfile):

anderser

from django.shortcuts import render_to_response, get_object_or_404, get_list_or_404
from django.http import HttpResponseRedirect, Http404, HttpResponse
from django.template import RequestContext
from django.core.cache import cache
import iso8601

from pollster import Pollster

def presidentmatch(request):

anderser

from local_settings import GOOGLE_USER, GOOGLE_PWD
from googleanalytics import Connection
from datetime import date

#googleanalytics is found here: https://github.com/clintecker/python-googleanalytics/

gaaccounts = {
              'myaccountname': '1234567',

              }

anderser

api = client.InstagramAPI(client_id=CONFIG['client_id'], client_secret=CONFIG['client_secret'])

instagen, pages = api.tag_recent_media(30, 0, "thetag", max_pages=1, return_json=True)

videos = []
for item in instagen:
    if "videos" in item:
        videos.append(
            {'user': item['user']['username'], 
            'videourl': item['videos']['standard_resolution']

anderser

import agate

class Streaks(agate.Computation):
    """
    Computes the streaks of consecutive values in a column.
    Each streak will be given an increasing
    integer value so that you can group by this later to
    find longest consecutive streak.
    """

anderser

from __future__ import print_function
from builtins import range
from airflow.operators import PythonOperator, PostgresOperator, DummyOperator
from airflow.models import DAG
from datetime import datetime, timedelta

import time
from pprint import pprint

seven_days_ago = datetime.combine(

anderser

1. Fetch some json/XML file from an external API
2. Store the file on S3 (as a backup) with a file name with a i.e. a timestamp. Pass the filename on to next task
3. Read the json/xml from S3 and into some table structure (pandas, agate etc) and change field types etc.
4. Store the table in a postgres database in a temp table
5. Compare the temp table to a "main" table and see if there are changes (some SQL diff). Find out which records have to be added/removed/updated in the "main" table.
6. If nothing has changed, abort everything. If it has, pass on which records are new, deleted and updated.
7. a) insert new records in main table, alert newsroom on slack of new items.b) delete items in main table not in temp-table. Alert via slack. c) Update records in main table, alert via slack
8. The end

anderser

WITH ais AS
  (SELECT *
   FROM aisdata.bronnbaater_resampled_min_lowspeed),
     lokaliteter AS
  (SELECT *
   FROM fishandfjord.fiskeridir_alle_lokaliteter
   WHERE loknr = '11763')
SELECT truncdate,
       loknr,
       lokalitet,