Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
import pandas as pd
from hdfs import InsecureClient
import os
# ====== Connection ======
# Connecting to Webhdfs by providing hdfs host ip and webhdfs port (50070 by default)
client_hdfs = InsecureClient('http://' + os.environ['IP_HDFS'] + ':50070')
# ====== Writing files ======
# Creating a simple pandas DataFrame
liste_hello = ['hello1','hello2']
liste_world = ['world1','world2']
df = pd.DataFrame(data = {'hello' : liste_hello, 'world': liste_world})
# Writing Dataframe to hdfs
with client_hdfs.write('/user/hdfs/wiki/helloworld.csv', encoding = 'utf-8') as writer:
df.to_csv(writer)
# ====== Reading files ======
with client_hdfs.read('/user/hdfs/wiki/helloworld.csv', encoding = 'utf-8') as reader:
df = pd.read_csv(reader,index_col=0)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment