Skip to content

Instantly share code, notes, and snippets.

View naturalett's full-sized avatar

Lidor Ettinger naturalett

View GitHub Profile
@naturalett
naturalett / tasks.txt
Created June 6, 2023 05:46
List of tasks to run on AWS Batch
date
echo "Hello world 1"
date
echo "Hello world 2"
echo "Hello world 3"
echo "Hello world 4"
@naturalett
naturalett / airflow_tasks_batch_based_s3.py
Created June 5, 2023 22:44
Airflow running bash tasks on AWS Batch Operator based on tasks.txt
from datetime import datetime, timedelta
from airflow import DAG
from airflow.operators.bash import BashOperator
from airflow.providers.amazon.aws.operators.batch import AwsBatchOperator
from airflow.providers.amazon.aws.operators.s3_copy_object import S3CopyObjectOperator
from airflow.providers.amazon.aws.operators.s3_list import S3ListOperator
from airflow.operators.python_operator import PythonOperator
import logging
import json
@naturalett
naturalett / bash_tasks_batch.py
Created June 5, 2023 22:25
Airflow running bash tasks on AWS Batch Operator
from datetime import datetime, timedelta
from airflow import DAG
from airflow.operators.bash import BashOperator
from airflow.providers.amazon.aws.operators.batch import AwsBatchOperator
NUM_TASKS = 500
default_args = {
'owner': 'airflow',
@naturalett
naturalett / airflow_bash_tasks_on_k8s.py
Created June 5, 2023 21:57
Airflow running bash tasks on k8s
from datetime import datetime, timedelta
from airflow import DAG
from airflow.operators.bash import BashOperator
from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator
NUM_TASKS = 500
default_args = {
'owner': 'airflow',
@naturalett
naturalett / airflow_bash.py
Last active June 5, 2023 21:56
Airflow running bash tasks
from datetime import datetime, timedelta
from airflow import DAG
from airflow.operators.bash import BashOperator
NUM_TASKS = 500
default_args = {
'owner': 'airflow',
'depends_on_past': False,
@naturalett
naturalett / iris_classification.py
Last active May 31, 2023 21:28
Iris Classification
from airflow import DAG
from airflow.operators.python_operator import PythonOperator
from datetime import datetime
import mysql.connector
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier
import joblib
from datetime import timedelta
import os
@naturalett
naturalett / slackSend.groovy
Created May 25, 2023 11:58
My First Pipeline Slack
pipeline {
agent any
stages {
stage('Clone') {
steps {
git branch: 'main', url: 'https://github.com/naturalett/hello-world.git'
}
}
}
post {
@naturalett
naturalett / Dockerfile
Last active January 6, 2024 16:39
Jenkins with docker engine
FROM jenkins/jenkins:2.387-jdk11
# ENV JAVA_OPTS="-Djenkins.install.runSetupWizard=false"
RUN jenkins-plugin-cli --plugins \
workflow-aggregator:596.v8c21c963d92d \
git:5.0.2 \
docker-workflow:563.vd5d2e5c4007f \
docker-plugin:1.3.0
@naturalett
naturalett / VCS-Docker-Agent-Build.groovy
Created May 14, 2023 20:37
Run a Containerized Jenkins Environment with Docker Agent for Seamless Integration!
pipeline {
agent {
docker {
image 'docker:19.03.12'
args '-v /var/run/docker.sock:/var/run/docker.sock'
}
}
stages {
stage('Clone') {
steps {
@naturalett
naturalett / k8s-pod-template.yaml
Created December 26, 2022 00:04
k8s-pod-template
apiVersion: v1
kind: Pod
metadata:
labels:
product: datascience
<% try { %>
<% build_label=(build_label) %>
<% } catch(Exception e) { %>
<% build_label='default' %>
<% } %>