saraengineer

from datasets import load_dataset, DatasetDict
from transformers import BertTokenizer, BertForSequenceClassification
from torch.utils.data import DataLoader
import torch
from transformers import Trainer, TrainingArguments, DataCollatorWithPadding
import logging

logging.disable(logging.WARNING)

def main():

saraengineer

version: '3.6'

volumes:
  zookeeper-data:
    driver: local
  zookeeper-log:
    driver: local
  kafka-data:
    driver: local

saraengineer

version: '3.6'

volumes:
  zookeeper-data:
    driver: local
  zookeeper-log:
    driver: local
  kafka-data:
    driver: local 
    

saraengineer

  version: '3.6'

  # ----------------- #
  # Apache Spark      #
  # ----------------- #
  spark-master:
    image: bitnami/spark:latest
    environment:
      - SPARK_MODE=master
      - SPARK_LOCAL_IP=spark-master