Skip to content

Instantly share code, notes, and snippets.

@Slach
Created December 27, 2023 14:24
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save Slach/b23b8d125ee0c6e7d48337522a4d9057 to your computer and use it in GitHub Desktop.
Save Slach/b23b8d125ee0c6e7d48337522a4d9057 to your computer and use it in GitHub Desktop.
reproduce ALTER TABLE FREEZE for encrypted over s3 disk for ReplicatedMergeTree
<clickhouse>
<listen_host replace="replace">0.0.0.0</listen_host>
<logger>
<console>1</console>
</logger>
<keeper_server>
<tcp_port>2181</tcp_port>
</keeper_server>
</clickhouse>
services:
minio:
image: docker.io/bitnami/minio:${MINIO_VERSION:-latest}
environment:
MINIO_ACCESS_KEY: access-key
MINIO_SECRET_KEY: it-is-my-super-secret-key
MINIO_DEFAULT_BUCKETS: 'clickhouse'
MINIO_ROOT_USER: access-key
MINIO_ROOT_PASSWORD: it-is-my-super-secret-key
healthcheck:
test: curl -sL http://localhost:9000/
interval: 10s
retries: 30
keeper:
image: ${CLICKHOUSE_KEEPER_IMAGE:-docker.io/clickhouse/clickhouse-keeper}:${CLICKHOUSE_KEEPER_VERSION:-latest-alpine}
volumes:
- ./clickhouse-keeper.xml:/etc/clickhouse-keeper/conf.d/clickhouse-keeper.xml
healthcheck:
test: bash -c 'if [[ "$$(echo 'ruok' | nc 127.0.0.1 2181)" == "imok" ]]; then exit 0; else exit 1; fi'
interval: 3s
timeout: 2s
retries: 5
start_period: 2s
clickhouse:
image: ${CLICKHOUSE_SERVER_IMAGE:-docker.io/clickhouse/clickhouse-server}:${CLICKHOUSE_SERVER_VERSION:-latest-alpine}
volumes:
- ./storage_configuration.xml:/etc/clickhouse-server/config.d/storage_configuration.xml
healthcheck:
test: clickhouse client -q "SELECT 1"
interval: 10s
timeout: 2s
retries: 30
start_period: 5s
links:
- keeper
- minio
depends_on:
minio:
condition: service_healthy
keeper:
condition: service_healthy
#!/usr/bin/env bash
set -xeo pipefail
docker-compose down
docker-compose up -d clickhouse
docker-compose exec clickhouse clickhouse-client -q "CREATE DATABASE test"
docker-compose exec clickhouse clickhouse-client -q "CREATE TABLE test.test (v UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/{database}/{table}','{replica}') ORDER BY tuple() SETTINGS storage_policy='s3_only'"
docker-compose exec clickhouse clickhouse-client -q "INSERT INTO TABLE test.test SELECT number FROM numbers(1000000)"
docker-compose exec clickhouse clickhouse-client -q "ALTER TABLE test.test FREEZE WITH NAME 'test'"
docker-compose exec clickhouse clickhouse-client -q "DROP TABLE test.test SYNC"
docker-compose exec clickhouse clickhouse-client -q "CREATE TABLE test.test_encrypted (v UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/{database}/{table}','{replica}') ORDER BY tuple() SETTINGS storage_policy='s3_only_encrypted'"
docker-compose exec clickhouse clickhouse-client -q "INSERT INTO TABLE test.test_encrypted SELECT number FROM numbers(1000000)"
docker-compose exec clickhouse clickhouse-client -q "ALTER TABLE test.test_encrypted FREEZE WITH NAME 'test'"
docker-compose exec clickhouse clickhouse-client -q "DROP TABLE test.test_encrypted SYNC"
<clickhouse>
<storage_configuration>
<disks>
<disk_s3>
<type>s3</type>
<endpoint>http://minio:9000/clickhouse/disk_s3/</endpoint>
<access_key_id>access-key</access_key_id>
<secret_access_key>it-is-my-super-secret-key</secret_access_key>
<!-- to avoid slow startup -->
<send_metadata>false</send_metadata>
</disk_s3>
<disk_s3_encrypted>
<type>encrypted</type>
<disk>disk_s3</disk>
<path>disk_s3_encrypted/</path>
<algorithm>AES_128_CTR</algorithm>
<key_hex id="0">00112233445566778899aabbccddeeff</key_hex>
<key_hex id="1">ffeeddccbbaa99887766554433221100</key_hex>
<current_key_id>1</current_key_id>
<!-- to avoid slow startup -->
<send_metadata>false</send_metadata>
</disk_s3_encrypted>
</disks>
<policies>
<s3_only>
<volumes>
<s3_only>
<disk>disk_s3</disk>
</s3_only>
</volumes>
</s3_only>
<s3_only_encrypted>
<volumes>
<s3_only_encrypted>
<disk>disk_s3_encrypted</disk>
</s3_only_encrypted>
</volumes>
</s3_only_encrypted>
</policies>
</storage_configuration>
<macros>
<cluster>cluster</cluster>
<shard>0</shard>
<replica>clickhouse</replica>
</macros>
<zookeeper>
<node index="1">
<host>keeper</host>
<port>2181</port>
</node>
</zookeeper>
</clickhouse>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment