databill86

## gpt-2-wikitext-103.py
# Copyright (c) 2019-present, Thomas Wolf.
# All rights reserved. This source code is licensed under the MIT-style license.
""" A very small and self-contained gist to train a GPT-2 transformer model on wikitext-103 """
import os
from collections import namedtuple
from tqdm import tqdm
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from ignite.engine import Engine, Events

## deployment-tool-ansible-puppet-chef-salt.md

      
              1 file
            
          
              0 forks
            
          
                0 comments
              
            
              0 stars
            
          
                databill86
                / deployment-tool-ansible-puppet-chef-salt.md
            
            
              Created
              March 28, 2019 14:37
                — forked from jaceklaskowski/deployment-tool-ansible-puppet-chef-salt.md
            
              
                Choosing a deployment tool - ansible vs puppet vs chef vs salt
              
          
    Requirements


no upfront installation/agents on remote/slave machines - ssh should be enough
application components should use third-party software, e.g. HDFS, Spark's cluster, deployed separately
configuration templating
environment requires/asserts, i.e. we need a JVM in a given version before doing deployment
deployment process run from Jenkins

Solution


## simple_python_datasource.py
from flask import Flask, request, jsonify, json, abort
from flask_cors import CORS, cross_origin

import pandas as pd

app = Flask(__name__)

cors = CORS(app)
app.config['CORS_HEADERS'] = 'Content-Type'

## dynamic_tasks.py
# Task primitives, allows pipeline execution using celery
@app.task
def dmap(it, callback, final=None):
    # Map a callback over an iterator and return as a group
    callback = subtask(callback)

    # Hack for mapping a chain to values, due to a bug where args are not copied in group creation
    if isinstance(callback, chain):
        if final:
            raise ValueError('task_processor: Cannot run reducer for dmap excecuted with a chain.')

## celery_task_monitor.py
import pickle
import threading
from Queue import Queue

import time
from bson import InvalidDocument

from celery.utils.log import get_task_logger

logger = get_task_logger(__name__)

## min-char-rnn.py
"""
Minimal character-level Vanilla RNN model. Written by Andrej Karpathy (@karpathy)
BSD License
"""
import numpy as np

# data I/O
data = open('input.txt', 'r').read() # should be simple plain text file
chars = list(set(data))
data_size, vocab_size = len(data), len(chars)

## xrpaway.py
# Requirement: pip install tweepy
import tweepy

# Credentials go here (generate at: https://apps.twitter.com)
auth = tweepy.OAuthHandler('consumer_key', 'consumer_secret')
auth.set_access_token('access_token', 'access_token_secret')

# Connect to Twitter
api = tweepy.API(auth)
	# Copyright (c) 2019-present, Thomas Wolf.
	# All rights reserved. This source code is licensed under the MIT-style license.
	""" A very small and self-contained gist to train a GPT-2 transformer model on wikitext-103 """
	import os
	from collections import namedtuple
	from tqdm import tqdm
	import torch
	import torch.nn as nn
	from torch.utils.data import DataLoader
	from ignite.engine import Engine, Events
	from flask import Flask, request, jsonify, json, abort
	from flask_cors import CORS, cross_origin

	import pandas as pd

	app = Flask(__name__)

	cors = CORS(app)
	app.config['CORS_HEADERS'] = 'Content-Type'
	# Task primitives, allows pipeline execution using celery
	@app.task
	def dmap(it, callback, final=None):
	# Map a callback over an iterator and return as a group
	callback = subtask(callback)

	# Hack for mapping a chain to values, due to a bug where args are not copied in group creation
	if isinstance(callback, chain):
	if final:
	raise ValueError('task_processor: Cannot run reducer for dmap excecuted with a chain.')
	import pickle
	import threading
	from Queue import Queue

	import time
	from bson import InvalidDocument

	from celery.utils.log import get_task_logger

	logger = get_task_logger(__name__)
	"""
	Minimal character-level Vanilla RNN model. Written by Andrej Karpathy (@karpathy)
	BSD License
	"""
	import numpy as np

	# data I/O
	data = open('input.txt', 'r').read() # should be simple plain text file
	chars = list(set(data))
	data_size, vocab_size = len(data), len(chars)
	# Requirement: pip install tweepy
	import tweepy

	# Credentials go here (generate at: https://apps.twitter.com)
	auth = tweepy.OAuthHandler('consumer_key', 'consumer_secret')
	auth.set_access_token('access_token', 'access_token_secret')

	# Connect to Twitter
	api = tweepy.API(auth)