load.bash

#!/bin/bash

# ClickHouse connection details
CLICKHOUSE_HOST="localhost"            # Replace with your ClickHouse host
CLICKHOUSE_PORT="9000"                 # Replace with your ClickHouse port if necessary
CLICKHOUSE_USER="default"              # Replace with your ClickHouse username
CLICKHOUSE_PASSWORD="your_password"    # Replace with your ClickHouse password
DB_NAME="your_database_name"           # Replace with your ClickHouse database
TABLE_NAME="your_table_name"           # Replace with your ClickHouse table

# Directory containing .json.gz files
DIRECTORY="path/to/your/json_gz_files"  # Replace with your directory path

# Loop through each .json.gz file in the directory
for file in "$DIRECTORY"/*.json.gz; do
    if [[ -f "$file" ]]; then
        echo "Processing $file..."
        
        # Unzip the file (will create a .json file)
        gunzip -c "$file" > "${file%.gz}"

        # Import the JSON file into ClickHouse
        clickhouse-client --host="$CLICKHOUSE_HOST" --port="$CLICKHOUSE_PORT" \
            --user="$CLICKHOUSE_USER" --password="$CLICKHOUSE_PASSWORD" \
            --query="INSERT INTO $DB_NAME.$TABLE_NAME FORMAT JSONAsObject" < "${file%.gz}"

        # Check if the import was successful
        if [[ $? -eq 0 ]]; then
            echo "Successfully imported ${file%.gz} into ClickHouse."
            # Optionally, delete the unzipped .json file after import to save space
            rm "${file%.gz}"
        else
            echo "Failed to import ${file%.gz}."
        fi
    else
        echo "No .json.gz files found in the directory."
    fi
done

echo "All files have been processed."