Skip to content

Instantly share code, notes, and snippets.

View TiGaI's full-sized avatar
🏠
Working from home

Junjie TiGaI

🏠
Working from home
5 followers · 2 following
View GitHub Profile
@TiGaI
TiGaI / init_callbacks_function.py
Created May 29, 2020 02:24
def init_callbacks(app, countFreq):
@app.callback(
[dash.dependencies.Output('frequency_word_bargraph', 'figure'),
dash.dependencies.Output('matplotlib-graph', 'src')],
[dash.dependencies.Input('range_frequency_number', 'value')])
def update_graph(value):
newGraph = countFreq[value[0]:value[1]]
wordcloud = WordCloud(height=500, width=500, background_color="white", contour_color='white', colormap="magma").generate_from_frequencies(newGraph)
buf = io.BytesIO() # in-memory files
@TiGaI
TiGaI / dashboard_layout.py
Created May 28, 2020 22:58
html_layout = '''
<!DOCTYPE html>
<html>
<head>
{%metas%}
<title>{%title%}</title>
<link type="text/css" rel="stylesheet" href="/static/css/font-awesome-4.1.0.min.css" />
<link type="text/css" rel="stylesheet" href="/static/css/bootstrap-3.1.1.min.css">
<link type="text/css" rel="stylesheet" href="/static/css/bootstrap-theme-3.1.1.min.css" />
<link type="text/css" rel="stylesheet" href="/static/css/layout.main.css" />
@TiGaI
TiGaI / dashboard.py
Created May 28, 2020 02:37
import plotly
import plotly.graph_objs as go
import pandas as pd
import numpy as np
import json
import nltk
#for NLP purpose
@TiGaI
TiGaI / Dockerfile-blog2
Created May 19, 2020 06:03
# VERSION 1.10.9
# AUTHOR: Matthieu "Puckel_" Roisil
# DESCRIPTION: Basic Airflow container
# BUILD: docker build --rm -t puckel/docker-airflow .
# SOURCE: https://github.com/puckel/docker-airflow
FROM python:3.6-slim-buster
LABEL maintainer="Puckel_"
# Never prompt the user for choices on installation/configuration of packages
@TiGaI
TiGaI / blog2_final.py
Created May 19, 2020 02:50
from airflow import DAG
from airflow.operators.python_operator import PythonOperator, BranchPythonOperator
from airflow.operators.dummy_operator import DummyOperator
import logging
import datetime
import twint
#directly import
# from sensors.gcs_bq_custom_sensor import GoogleCloudStorageBigQueryUpdateSensor
@TiGaI
TiGaI / checkForDuplication.py
Created May 19, 2020 02:33
class CheckBQDuplication(BaseOperator):
"""
Check if a specific table in BigQuery contains duplicated data after the load
"""
@apply_defaults
def __init__(
self,
dataset_name,
bigquery_table_name,
bigquery_table_key,
@TiGaI
TiGaI / branchingOperatorExample.py
Last active May 19, 2020 02:16
def checkingYesterdayTweet(bucket_name, project, credentials_path, **kwargs):
credentials = service_account.Credentials.from_service_account_file(credentials_path) if credentials_path else None
storage_client = storage.Client(project=project, credentials=credentials)
bucket_name = "airflowexample"
bucket = storage_client.get_bucket(bucket_name)
yesterday = datetime.datetime.today() - datetime.timedelta(days=1)
searchTerm = "coronavirus"
filename = f"tweet-{searchTerm}-{yesterday.strftime('%Y-%m-%d')}"
@TiGaI
TiGaI / blog2_example#2
Created May 18, 2020 23:30
from airflow import DAG
from google.cloud import storage
from google.oauth2 import service_account
from airflow.operators.python_operator import PythonOperator
from airflow.operators.dummy_operator import DummyOperator
from io import BytesIO, StringIO
import pandas as pd
import numpy as np
@TiGaI
TiGaI / GCS_BQ_Custom_Sensor.py
Created May 18, 2020 02:56
from airflow.operators.sensors import BaseSensorOperator
from airflow.utils.decorators import apply_defaults
from google.cloud import bigquery
from google.cloud import storage
from google.oauth2 import service_account
import datetime
class GoogleCloudStorageBigQueryUpdateSensor(BaseSensorOperator):
"""
See if the modified date of the big query dataset is less than modified date of the GCS files.
@TiGaI
TiGaI / blog2_example#1
Last active May 15, 2020 03:08
from airflow import DAG
from google.cloud import storage
from google.oauth2 import service_account
from airflow.operators.python_operator import PythonOperator
from io import BytesIO, StringIO
import pandas as pd
import numpy as np
from datetime import datetime
import logging