1ambda

- display_name: "ML Type 1 [ON_DEMAND]"
      description: "Type 1 [ON_DEMAND] - Basic (CPU 2-6, Memory 20 GiB)"
      kubespawner_override:
        image: ...
        node_selector:
          ...
          eks.amazonaws.com/capacityType: ON_DEMAND
        cpu_limit: 6
        cpu_guarantee: 2
        mem_limit: '20G'

1ambda

# 사용자 별 제공되는 Jupyter 컨테이너에 Jupyter Hub 리소스 타입별로 환경변수를 미리 세팅
SPARK_OPTS=--master=local[6] --driver-memory=20g

1ambda

# Client Mode: Driver JVM 이 이미 떠있는 상태에서 Driver JVM 메모리를 지정할 순 없음
spark = SparkSession.builder \
       ...
       .config("spark.driver.memory", "9g")
       .getOrCreate()

1ambda

trait Monad[F[_]] {
  def point[A](a: A): F[A]
  def bind[A, B](fa: F[A])(f: A => F[B]): F[B]

  def map[A, B](fa: F[A])(f: A => B): F[B] =
    bind(fa)(a => point(f(a))
  def join[A](ffa: F[F[A]): F[A] =
    bind(ffa)(fa => fa)
}
                     

1ambda

# 최초 10 줄만 출력합니다.
$ kafka-run-class kafka.tools.DumpLogSegments --files /var/lib/kafka/data/test-topic-0/00000000000000000000.index --print-data-log | head -n 10

Dumping /var/lib/kafka/data/test-topic-0/00000000000000000000.index
offset: 32 position: 17275
offset: 48 position: 33480
offset: 64 position: 49685
offset: 80 position: 65890
offset: 96 position: 82095
offset: 112 position: 98300

1ambda

test-topic-0  
├── 00000000000000000001.index  
├── 00000000000000000001.log  
├── 00000000000000000001.timeindex  
├── 00000000000000001018.index  
├── 00000000000000001018.log  
├── 00000000000000001018.timeindex  
├── 00000000000000002042.index  
├── 00000000000000002042.log  
├── 00000000000000002042.timeindex

1ambda

// Some code$ kafka-producer-perf-test --topic test-topic --num-records 500000 --record-size 100 --throughput 10000  --producer-props bootstrap.servers=broker:9092

$ ls /var/lib/kafka/data/ | grep test
test-topic-0

$ ls /var/lib/kafka/data/test-topic-0
00000000000000000000.index  00000000000000000000.log  00000000000000000000.timeindex  leader-epoch-checkpoint

1ambda

kafka-topics --describe --zookeeper zookeeper:2181 --topic __consumer_offsets
Topic: __consumer_offsets       TopicId: wWPS-BD9TSuZHybvPwpomg PartitionCount: 50      ReplicationFactor: 1    Configs: compression.type=producer,cleanup.policy=compact,segment.bytes=104857600
        Topic: __consumer_offsets       Partition: 0    Leader: 1       Replicas: 1     Isr: 1
        Topic: __consumer_offsets       Partition: 1    Leader: 1       Replicas: 1     Isr: 1
        ....
        Topic: __consumer_offsets       Partition: 49   Leader: 1       Replicas: 1     Isr: 1

1ambda

$ cd /var/lib/kafka/data

$ ls

__consumer_offsets-0   __consumer_offsets-15  __consumer_offsets-21  __consumer_offsets-28  __consumer_offsets-34  __consumer_offsets-40  __consumer_offsets-47  __consumer_offsets-9
__consumer_offsets-1   __consumer_offsets-16  __consumer_offsets-22  __consumer_offsets-29  __consumer_offsets-35  __consumer_offsets-41  __consumer_offsets-48  _schemas-0
__consumer_offsets-10  __consumer_offsets-17  __consumer_offsets-23  __consumer_offsets-3   __consumer_offsets-36  __consumer_offsets-42  __consumer_offsets-49  cleaner-offset-checkpoint
__consumer_offsets-11  __consumer_offsets-18  __consumer_offsets-24  __consumer_offsets-30  __consumer_offsets-37  __consumer_offsets-43  __consumer_offsets-5   log-start-offset-checkpoint
__consumer_offsets-12  __consumer_offsets-19  __consumer_offsets-25  __consumer_offsets-31  __consumer_offsets-38  __consumer_offsets-44  __consumer_offsets-6   meta.properties
__consumer_offsets-13  __consumer_offsets-2   __consumer_offsets-26  __consumer_offse

1ambda

$ cd /etc/kafka

$ cat kafka.properties | grep log.dir
log.dirs=/var/lib/kafka/data