ttedeschi

import pandas as pd
from pyspark.sql.types import *
from pyspark.sql import Row
import json
from pyspark.sql import SQLContext
from CMSSpark.spark_utils import dbs_tables, phedex_tables

aformat = "org.apache.avro.mapreduce.AvroKeyInputFormat"
akey = "org.apache.avro.mapred.AvroKey"
awrite = "org.apache.hadoop.io.NullWritable"

ttedeschi

tosca_definitions_version: tosca_simple_yaml_1_0

imports:
  - indigo_custom_types: https://raw.githubusercontent.com/dodas-ts/dodas-templates/master/tosca-types/dodas_custom_apps/spark_type.yml
description: TOSCA template for a complete CMS computing cluster on top of K8s orchestrator

topology_template:

  inputs:

ttedeschi

FROM ubuntu

# Install prerequisites
RUN apt-get update && apt-get install -y \
curl

COPY ./process_exporter /bin/process-exporter

# Run the process-exporter command by default when the container starts.
ENTRYPOINT ["/bin/process-exporter"]

ttedeschi

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: prometheus
  namespace: monitoring
spec:
  replicas: 1
  template:
    metadata:
      labels:

ttedeschi

# Source: prometheus-adapter/templates/custom-metrics-apiserver-service-account.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    app: prometheus-adapter
    chart: prometheus-adapter-2.3.1
    release: my-release
    heritage: Helm
  name: my-release-prometheus-adapter

ttedeschi

apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    app: prometheus-adapter
  name: prometheus-adapter
---

apiVersion: v1
kind: ConfigMap

ttedeschi

apiVersion: autoscaling/v2beta2
kind: HorizontalPodAutoscaler
metadata:
  name: httpgo-hpa
  namespace: default
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: httpgo

ttedeschi

apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    app: prometheus-adapter
    chart: prometheus-adapter-2.3.1
  name: prometheus-adapter
---

apiVersion: v1

ttedeschi

condorHost: <master public IP>
ccbHost: <master public IP>
proxyCacheHost: <master private IP>
ttsCache:
    image: dodasts/tts-cache
    tag: v0.1.3-k8s-12
    iamToken: < your valid access token >
    iamClientId: CHANGEME 
    iamClientSecret: CHANGEME 
htcSchedd:

ttedeschi

import argparse
import time

import htcondor
import re

from prometheus_client import start_wsgi_server
from prometheus_client import REGISTRY, make_wsgi_app
from prometheus_client.core import GaugeMetricFamily
from wsgiref.simple_server import make_server