Created
April 19, 2021 21:24
-
-
Save azat/4813489828162e6c2ce131963c6a1acb to your computer and use it in GitHub Desktop.
clickhouse-min_bytes_to_use_mmap_io-issue
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?xml version="1.0"?> | |
<yandex> | |
<logger> | |
<level>trace</level> | |
<console>true</console> | |
</logger> | |
<tcp_port>9000</tcp_port> | |
<path>./</path> | |
<mark_cache_size>0</mark_cache_size> | |
<!-- One cannot overrid it via cli options, since it is reloadable --> | |
<max_server_memory_usage>1000000000</max_server_memory_usage> <!-- 1G --> | |
<!-- Sync with RSS earlier --> | |
<asynchronous_metrics_update_period_s>1</asynchronous_metrics_update_period_s> | |
<users> | |
<default> | |
<password></password> | |
<networks> | |
<ip>::/0</ip> | |
</networks> | |
<profile>default</profile> | |
<quota>default</quota> | |
<access_management>1</access_management> | |
</default> | |
</users> | |
<profiles> | |
<default/> | |
</profiles> | |
<quotas> | |
<default /> | |
</quotas> | |
</yandex> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) | |
# shellcheck source=../shell_config.sh | |
export CLICKHOUSE_DATABASE=default | |
. "$CUR_DIR"/../shell_config.sh | |
server_config="$CUR_DIR/$(basename "${BASH_SOURCE[0]}" .sh).config.xml" | |
server_log="$CUR_DIR/$(basename "${BASH_SOURCE[0]}" .sh).server.log" | |
server_pid= | |
server_port= | |
server_path="$(mktemp -d clickhouse.XXXXXX)" | |
function client() { $CLICKHOUSE_CLIENT --host 127.1 --port "$server_port" "$@"; } | |
function cleanup() | |
{ | |
kill -9 $server_pid | |
echo "Test failed. Server log:" | |
cat "$server_log" | |
rm -rf "$server_log" "$server_path" | |
exit 1 | |
} | |
# wait until server will start to listen (max 30 seconds) | |
function server_discover_listen_port() | |
{ | |
server_port= | |
i=0 retries=300 | |
while [[ -z $server_port ]] && [[ $i -lt $retries ]]; do | |
server_port=$(lsof -n -a -P -i tcp -s tcp:LISTEN -p $server_pid 2>/dev/null | awk -F'[ :]' '/LISTEN/ { print $(NF-1) }') | |
((++i)) | |
sleep 0.1 | |
done | |
if [[ -z $server_port ]]; then | |
echo "Cannot wait for LISTEN socket" >&2 | |
exit 1 | |
fi | |
} | |
# wait for the server to start accepting tcp connections (max 30 seconds) | |
function server_wait_accept() | |
{ | |
i=0 retries=300 | |
while ! client --format Null -q 'select 1' 2>/dev/null && [[ $i -lt $retries ]]; do | |
sleep 0.1 | |
done | |
if ! client --format Null -q 'select 1'; then | |
echo "Cannot wait until server will start accepting connections on <tcp_port>" >&2 | |
exit 1 | |
fi | |
} | |
function server_start() | |
{ | |
local server_opts=( | |
"--config-file=$server_config" | |
"--" | |
"--path=$server_path" | |
# to avoid multiple listen sockets (complexity for port discovering) | |
"--listen_host=127.1" | |
# we will discover the real port later. | |
"--tcp_port=0" | |
"--shutdown_wait_unfinished=0" | |
) | |
CLICKHOUSE_WATCHDOG_ENABLE=0 $CLICKHOUSE_SERVER_BINARY "${server_opts[@]}" >& "$server_log" & | |
server_pid=$! | |
trap cleanup EXIT | |
server_discover_listen_port | |
server_wait_accept | |
} | |
function client_run() | |
{ | |
client -nm -q " | |
CREATE TABLE data_01830 ( | |
key UInt64 CODEC(NONE), | |
value String CODEC(NONE) | |
) | |
Engine=MergeTree() | |
ORDER BY key | |
SETTINGS min_bytes_for_wide_part=0; | |
SYSTEM STOP MERGES data_01830; | |
INSERT INTO data_01830 SELECT number, toString(number) FROM numbers(1e6); | |
" | |
# duplicate parts (using hardlinks) | |
for i in {1..5}; do | |
client -q "ALTER TABLE data_01830 ATTACH PARTITION tuple() FROM data_01830" | |
done | |
# expand hardlinks | |
client -nm -q " | |
SYSTEM START MERGES data_01830; | |
OPTIMIZE TABLE data_01830 FINAL; | |
" | |
client -q "SELECT ignore(*) FROM data_01830" --format Null --min_bytes_to_use_mmap_io=0 | |
# since mmap() does not accounted in the memory tracker | |
# once memory tracker will be synced with RSS and reach max_server_memory_usage | |
# any subsequent allocation will fail, so this query will eventually fail. | |
if client -q "SELECT ignore(*) FROM data_01830" --format Null --min_bytes_to_use_mmap_io=1 2>/dev/null; then | |
echo "SELECT with min_bytes_to_use_mmap_io==0 should fail" >&2 | |
exit 1 | |
fi | |
return 0 | |
} | |
function main() | |
{ | |
server_start | |
client_run || exit 1 | |
trap '' EXIT | |
kill -9 $server_pid | |
rm -f "$server_log" | |
} | |
main "$@" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment