[Wheat detection] - Tiny Yolo implementation

tiny_yolo-wheat_detection.ipynb

{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install gdown","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!git clone https://github.com/ultralytics/yolov3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cd yolov3 && pip install -r requirements.txt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torchvision\n\n!pip install torchvision==0.4.0\ntorchvision.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DIR_INPUT = '/kaggle/input/global-wheat-detection'\nDIR_TRAIN = f'{DIR_INPUT}/train'\nDIR_TEST = f'{DIR_INPUT}/test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(DIR_INPUT+'/train.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re \n\ntrain_df['x'] = -1\ntrain_df['y'] = -1\ntrain_df['w'] = -1\ntrain_df['h'] = -1\n\ndef expand_bbox(x):\n    r = np.array(re.findall(\"([0-9]+[.]?[0-9]*)\", x))\n    if len(r) == 0:\n        r = [-1, -1, -1, -1]\n    return r\n\ntrain_df[['x', 'y', 'w', 'h']] = np.stack(train_df['bbox'].apply(lambda x: expand_bbox(x)))\ntrain_df.drop(columns=['bbox'], inplace=True)\ntrain_df['x'] = train_df['x'].astype(np.float)\ntrain_df['y'] = train_df['y'].astype(np.float)\ntrain_df['w'] = train_df['w'].astype(np.float)\ntrain_df['h'] = train_df['h'].astype(np.float)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['x'] = train_df['x']/train_df['width']\ntrain_df['y'] = train_df['y']/train_df['height']\ntrain_df['w'] = train_df['w']/train_df['width']\ntrain_df['h'] = train_df['h']/train_df['height']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['label'] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.sample()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mkdir -p data data/labels\ntrain_df.groupby('image_id').apply(\n    lambda df: \n    df[['label','x','y','w','h']]\\\n    .to_csv('data/labels/{}.txt'.format(df.image_id.iloc[0]), sep=' ', header=False, index=False, float_format='%.6f')\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cat data/labels/00333207f.txt\n# !rm data -R","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! rm yolov3/data/wheat -R","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cp -a ../input/global-wheat-detection/train data/images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!mv data yolov3/data/wheat","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pd.DataFrame([\n#     train_df.image_id.unique()\n# ])\nseed=42\nfrom sklearn.model_selection import train_test_split\nprint(\"Cantidad de imagenes: \", train_df.image_id.nunique())\n\ntrain, validation = train_test_split(train_df.image_id.unique(),random_state=seed, test_size=0.2)\nprint(\"Cantidad de imagenes en train: \", len(train))\nprint(\"Cantidad de imagenes en validation: \", len(validation))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.Series(['wheat']).to_csv('yolov3/data/wheat_1cls.names', index=False, header=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cat yolov3/data/wheat_1cls.names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_files = pd.DataFrame(train[0:6], columns=['filename'])\ntrain_files['path'] = './data/wheat/images/'\ntrain_files['extension'] = '.jpg'\ntrain_files['absolute_name'] = train_files.path + train_files.filename + train_files.extension\ntrain_files.absolute_name.to_csv('yolov3/data/example_wheat.txt', header=False, index=False, sep=' ')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_files = pd.DataFrame(train, columns=['filename'])\ntrain_files['path'] = './data/wheat/images/'\ntrain_files['extension'] = '.jpg'\ntrain_files['absolute_name'] = train_files.path + train_files.filename + train_files.extension\ntrain_files.absolute_name.to_csv('yolov3/data/wheat_train.txt', header=False, index=False, sep=' ')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation_files = pd.DataFrame(validation, columns=['filename'])\nvalidation_files['path'] = './data/wheat/images/'\nvalidation_files['extension'] = '.jpg'\nvalidation_files['absolute_name'] = train_files.path + train_files.filename + train_files.extension\nvalidation_files.absolute_name.to_csv('yolov3/data/wheat_validation.txt', header=False, index=False, sep=' ')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cat yolov3/data/example_wheat.txt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"!cat yolov3/data/wheat_train.txt\n!cat yolov3/data/wheat_validation.txt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('yolov3/data/wheat.data','w') as f: \n    f.write(\"classes=1\\n\")\n    f.write(\"train=./data/wheat_train.txt\\n\") # use example\n    f.write(\"valid=./data/wheat_validation.txt\\n\") # use example\n    f.write(\"names=./data/wheat_1cls.names\\n\")\n    f.write(\"backup=backup/\\n\")\n    f.write(\"eval=wheat\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cat yolov3/data/wheat.data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !cd yolov3/weights && gdown https://drive.google.com/uc?id=16lYS4bcIdM2HdmyJBVDOvt3Trx6N3W2R","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cd yolov3/weights && gdown https://drive.google.com/uc?id=1CCF-iNIIkYesIDzaPvdwlcf7H9zSsKZQ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls yolov3/weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls yolov3/cfg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# !rm yolov3/weights/1CCF-iNIIkYesIDzaPvdwlcf7H9zSsKZQ  yolov3/weights/yolov3-spp.weights  yolov3/weights/best.pt\t yolov3/weights/last.pt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n!cd yolov3 && python train.py --data data/wheat.data --cfg cfg/yolov3-tiny-1cls.cfg --weights weights/yolov3-tiny.weights --epochs 20\n# in case of continue learning use last.pt\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1st - 20 epochs - GIoU\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import FileLink\n\nFileLink(r'yolov3/weights/best.pt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FileLink(r'yolov3/weights/last.pt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !cp -r yolov3/data/wheat/images yolov3/data/samples/wheat","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cd yolov3 && python test.py --data data/wheat.data --cfg cfg/yolov3-tiny-1cls.cfg --weights weights/best.pt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"! cd yolov3 && python3 detect.py --cfg cfg/yolov3-tiny-1cls.cfg --weights weights/best.pt --names data/wheat_1cls.names --source='data/wheat/images' --save-txt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cp -a ../input/global-wheat-detection/test yolov3/data/wheat/test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! cd yolov3 && python3 detect.py --cfg cfg/yolov3-tiny-1cls.cfg --weights weights/best.pt --names data/wheat_1cls.names --source='data/wheat/test' --save-txt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls yolov3/output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cat yolov3/output/2fd875eaa.jpg.txt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image\nim = Image.open(\"yolov3/output/2fd875eaa.jpg\")\nim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# with open('yolov3/results.txt','r') as f:\n#     while True:\n#         line = f.readline()\n#         print(line)\n#         if not line:\n#             break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image\nim = Image.open(\"yolov3/results.png\")\nim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def format_prediction_string(boxes, scores):\n    pred_strings = []\n    for j in zip(scores, boxes):\n        pred_strings.append(\"{0:.4f} {1} {2} {3} {4}\".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))\n    return \" \".join(pred_strings)\n\ndef get_boxes(arr):\n    print(arr.reshape[-1,4,1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from glob import glob\n\nli=[]\n\nfor filename in glob('./yolov3/output/*.txt'):\n    id_file = filename.split(\"/\")[-1].split('.')[0]\n    df = pd.read_csv(filename,  header=None, usecols=[0,1,2,3,5], delimiter=' ')\n    df.columns=[\"x\",\"y\",\"w\",\"h\",\"score\"]\n    df['image_id'] = id_file\n    df = df[['image_id', 'score', 'x','y','w','h']]\n    li.append(df)\n\nli = pd.concat(li, ignore_index=True)\n\nresult = pd.DataFrame()\nresult['image_id'] = li.image_id\nresult['PredictionString'] = li.score.round(4).astype('str') +\" \"+li.x.astype('str')+\" \" + li.y.astype('str')+\" \" + li.h.astype('str') +\" \"+ li.w.astype('str')\n\nresult.to_csv('./20epochs-tinyyolov3_results.csv', header=True, sep=',', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\n","execution_count":233,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.read_csv('./20epochs-tinyyolov3_results.csv').to_csv('../submission.csv', index=False)","execution_count":236,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}

x & y should be "centered"