Chestermozhao

# -*- coding: utf-8 -*-
from elasticsearch import Elasticsearch
from const import (
    ES_HOST,
    ES_PORT
)

es = Elasticsearch(host=ES_HOST, port=ES_PORT)
es = Elasticsearch()

Chestermozhao

# -*- coding: utf-8 -*-
from elasticsearch import Elasticsearch
from elasticsearch import helpers
from const import (
    ES_HOST,
    ES_PORT
)

es = Elasticsearch(host= ES_HOST, port= ES_PORT)
es = Elasticsearch()

Chestermozhao

# -*- coding: utf-8 -*-
from elasticsearch import Elasticsearch
from elasticsearch import helpers
from const import (
    ES_HOST,
    ES_PORT
)

es = Elasticsearch(host= ES_HOST, port= ES_PORT)
es = Elasticsearch()

Chestermozhao

# -*- coding: utf-8 -*-
from elasticsearch import Elasticsearch
from const import (
    ES_HOST,
    ES_PORT
)

es = Elasticsearch(host= ES_HOST, port= ES_PORT)
es = Elasticsearch()

Chestermozhao

#!/bin/bash
export PYTHONPATH=<workplace>
/Users/bill/venv/bin/python3 subscribing_channels.py

Chestermozhao

# Importing modules
from datetime import timedelta, datetime
from airflow import DAG
from airflow.operators.bash_operator import BashOperator


# Setting default arguments
default_args = {
    "owner": "Chester mo",
    "start_date": datetime(2021, 1, 1, 0, 0),

Chestermozhao

{
    # channel_name: youtube channel name
    # channel_id: you need visit youtube, and copy the channel_id from url
    # like as `https://www.youtube.com/channel/UC-QDfvrRIDB6F0bIO4I4HkQ`
    # channel name is Pretty Printed(you can define by yourself, we only use channel_id to crawling)
    # channel id is UC-QDfvrRIDB6F0bIO4I4HkQ
    # please delete comment above, when you running airflow
    "channel_name": "channel_id",
}

Chestermozhao

import json
from .mongo import collection_sub_channel


def init_channel_info(name, _id):
    init_data = {
        "channel_name": name,
        "channel_id": _id,
        "previous_title": "",
        "previous_link": "",

Chestermozhao

import pymongo

# init mongo config
client = pymongo.MongoClient("mongodb://localhost:27017/")
db = client["youtube"]
collection_sub_channel = db["sub_channel"]

Chestermozhao

from .mongo import collection_sub_channel


def check_and_update_record(mode, **context):
    if mode == "check":
        sub_channels = collection_sub_channel.find()
        sub_channels = list(sub_channels) if sub_channels else ""
        return sub_channels
    elif mode == "update":
        print("Saving latest youtube information..")