conceptual_test.py

from db_model.models.content import RawArticle
from db_model.models.source import Source
from engine.celery import app, BaseTask
from engine.resources.vimeo_service import fetch_user_videos
from engine.resources.youtube_service import get_youtube_uploads
from time import strptime
from engine.utils.lock import FetchLock
import requests
# 2015-08-03T22:16:40+00:00
date_format = '%Y-%m-%dT%H:%M:%S+00:00'
dt2 = '%Y-%m-%dT%H:%M:%S+00:00'
def fetch_vimeo(source_id):
    source = Source.get_one_by_id(source_id)
    v = RawArticle.get_last_for_source_id(source.id)
    accessToken = source.credential.data.get('access-token', None);
    videos = fetch_user_videos(source.external_id, accessToken)
    videos.reverse()
    raw_articles = []
    for video in videos:
        if v is None or strptime(video['created_time'], date_format) > \
                strptime(v.data['created_time'], date_format):
            raw_articles.append(RawArticle(data=video, source=source))
    if RawArticle.save_many(raw_articles):
        source.register_fetch_success()
        source.save()
    else:
        source.register_fetch_error(Exception('something horrible happened'))
            source.save()
def fetch_youtube(source_id):
    source = Source.get_one_by_id(source_id)
    last_article = RawArticle.get_last_for_source_id(source.id)
    last_client_id = last_article.client_id if last_article else None
    articles = []
    api_key = source.credential.data.get("api_key", None)
    feed_data = get_youtube_uploads(source.external_id, api_key)
    filtered_items = []
    for item in feed_data["items"]:
        if item["snippet"]["resourceId"]["videoId"] == last_client_id:
            break
        filtered_items.append(item)
    ras=[]
    for item in reversed(filtered_items):
        ras.append(RawArticle(data=item, source=source, client_id=item["snippet"]["resourceId"]["videoId"]))
    if RawArticle.save_many(ras):
        source.register_fetch_success()
        source.save()
    else:
        source.register_fetch_error(Exception('something horrible happened'))
def get_youtube_uploads(channel_id, api_key):
    params = {
        "part": "id,snippet,contentDetails",
        "playlistId": channel_id,
        "key": api_key
    }
    res = requests.get('https://www.googleapis.com/youtube/v3/playlistItems', params=params)
    if not res.ok:
        raise (Exception(res.text))
    return res.json()