custom_container_data_s3_update.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "77ac98c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import yaml\n",
    "import sagemaker\n",
    "import boto3\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "SETTING_FILE_PATH = \"../../config/settings.yaml\"\n",
    "DATA_FOLDER_PATH = \"../avazu-ctr-prediction\"\n",
    "\n",
    "# AWS リソース設定\n",
    "with open(SETTING_FILE_PATH) as file:\n",
    "    aws_info = yaml.safe_load(file)\n",
    "        \n",
    "sess = sagemaker.Session()\n",
    "role = aws_info['aws']['sagemaker']['role']\n",
    "bucket = aws_info['aws']['sagemaker']['s3bucket']\n",
    "region = aws_info['aws']['sagemaker']['region']\n",
    "\n",
    "sm = boto3.client('sagemaker')\n",
    "s3 = boto3.client('s3')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0d4ee661",
   "metadata": {},
   "outputs": [],
   "source": [
    "# train, validation, test データを用意\n",
    "df_train = pd.read_csv(os.path.join(DATA_FOLDER_PATH, \"train\"), dtype=\"object\")\n",
    "df_train, df_test = train_test_split(df_train, train_size=0.8, random_state=0, shuffle=True)\n",
    "df_train, df_validation = train_test_split(df_train, train_size=0.7, random_state=0, shuffle=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3224d1bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# local mode用にローカル環境にデータを保存\n",
    "df_train.to_csv('train.csv', index=False)\n",
    "df_validation.to_csv('validation.csv', index=False)\n",
    "df_test.to_csv('test.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fca297b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# S3にアップロード\n",
    "prefix = 'custom-container-training'\n",
    "\n",
    "train_file = \"train.csv\"\n",
    "validation_file = \"validation.csv\"\n",
    "test_file = \"test.csv\"\n",
    "\n",
    "df_train.to_csv(train_file, index=False)\n",
    "df_validation.to_csv(validation_file, index=False)\n",
    "df_test.to_csv(test_file, index=False)\n",
    "\n",
    "s3_resource_bucket = boto3.Session().resource(\"s3\").Bucket(bucket)\n",
    "\n",
    "s3_resource_bucket.Object(os.path.join(prefix, \"train\", train_file)).upload_file(train_file)\n",
    "s3_resource_bucket.Object(os.path.join(prefix, \"validation\", validation_file)).upload_file(validation_file)\n",
    "s3_resource_bucket.Object(os.path.join(prefix, \"test\", test_file)).upload_file(test_file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fc45fc28",
   "metadata": {},
   "outputs": [],
   "source": [
    "output_location = f\"s3://{bucket}/{prefix}/output\"\n",
    "\n",
    "s3_train_data = f\"s3://{bucket}/{prefix}/train/{train_file}\"\n",
    "s3_validation_data = f\"s3://{bucket}/{prefix}/validation/{validation_file}\"\n",
    "s3_test_data = f\"s3://{bucket}/{prefix}/test/{test_file}\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}