Skip to content

Instantly share code, notes, and snippets.

@ehzawad

ehzawad/json_to_csv_snips_NLU.py

Created January 15, 2023 11:49
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save ehzawad/3ed7a4b396f3b49a957facada6c86316 to your computer and use it in GitHub Desktop.
Save ehzawad/3ed7a4b396f3b49a957facada6c86316 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
json_to_csv_snips_NLU.py
import pandas as pd
import time
import io
import json
from langdetect import detect
from snips_nlu import SnipsNLUEngine
import time
# with io.open("/home/custom-bot/sms/email_dataset.json") as f:
# sample_dataset = json.load(f)
with io.open("/home/ehz/Downloads/data_only_bn_new.json") as f:
sample_dataset_bn = json.load(f)
# nlu_engine = SnipsNLUEngine()
# nlu_engine.fit(sample_dataset)
nlu_engine_bn = SnipsNLUEngine()
nlu_engine_bn.fit(sample_dataset_bn)
RESPONSE_BN = {"SMS_balance_Q": "এস এম এস চেক করে কিভাবে"}
# RESPONSE_EN = {"prepaid_SMS_bundle_q": "Dear Customer, please dial *123*2*7*1# to purchase 100 SMS with 3 days validity at BDT5 (including SD, VAT and SC). To check remaining SMS"}
# creating a Flask app
#app = Flask(__name__)
def isASCII(data):
try:
data.encode().decode('ASCII')
except UnicodeDecodeError:
return False
else:
return True
#unclean_data = pd.read_csv('/home/custom-bot/sms/wow_call_6months.csv')
# print(unclean_data)
# print(unclean_data.columns.tolist())
# print(unclean_data['Subject'])
# super_clean_data = unclean_data[[' Subject',' Customer-Email']].dropna(axis=0, how='any')
#super_clean_data = unclean_data['Customer SMS'].dropna(axis=0, how='any')
# print(super_clean_data)
# super clean data to text
# super_clean_data.to_csv(r'/home/ehz/Downloads/super_clean_data.txt', header=None, index = False, sep='\t', mode='a')
#super_clean_data.to_csv(r'/home/custom-bot/sms/wow_call_6months.txt', header=None, index = False, mode='a')
f = open('/home/ehz/Downloads/test_data.txt', 'r')
# print(f.readlines())
# count = 0
#
# for line in f:
# count += 1
# print(base64.b64decode(line))
# print("{}{}".format(count, line.strip()))
def home(text):
f = open("/home/ehz/results_SMS_results_Q.csv", "a")
fl = open("/home/ehz/results_SMS_results_Q.log", "a")
req=1
if(req):
#parsing = nlu_engine.parse("Minute bundle kine kivabe Robi te")
#data = request.get_json()
#data = "hello world"
#lang = detect(data)
data = text
print(str(data))
lang = detect(data)
# parsing = nlu_engine.parse(str(data))
# print(parsing)
# print(parsing['intent']['intentName'])
# intent = parsing['intent']['intentName']
# probability = parsing['intent']['probability']
if(isASCII(data)):
pass
# if(intent is None):
# intent='None'
# if(probability is None):
# probability='None'
# if(intent=='None'):
# if(data.find('mb')>-1 or data.find('GB')>-1 or data.find('gb')>-1):
# intent='email_clm_data_q'
# fl.write(str(intent)+'|email_clm_data_q)|'+str(probability)+'|'+str(data)+'\n')
# fl.close()
# if(probability<0.40):
# if(data.find('mb')>-1 or data.find('GB')>-1 or data.find('gb')>-1):
# intent='email_CLM_data_q'
# fl.write(str(intent)+'|email_clm_data_q)|'+str(probability)+'|'+str(data)+'\n')
# fl.close()
# f.write(str(data).rstrip()+'|email|'+intent+'|'+str(probability)+'\n')
# f.close()
# if(lang=='en'):
# if(intent in RESPONSE_EN):
# return {'text':RESPONSE_EN[intent]}
# else:
# return {'text':'no intent found'}
# else:
# if(intent in RESPONSE_BN):
# return {'text':RESPONSE_BN[intent]}
# else:
# return {'text':'kono intent paoya jaini'}
else:
print("Using BN Model.")
parsing = nlu_engine_bn.parse(str(data))
print(parsing)
intent = parsing['intent']['intentName']
probability = parsing['intent']['probability']
if(intent is None):
intent='None'
if(probability is None):
probability='None'
# if(intent=='None'):
# if('এয়াটেল' in str(data) or 'এয়ারটেলে' in str(data) or 'এয়ারটেল' in str(data)):
# intent='email_airtel_related_query_in_robi_helpline'
# if('জি.বি' in str(data) or 'এমবি' in str(data) or 'জিবির' in str(data) or 'এমবি' in str(data) or 'এম্বি' in str(data) or 'জিবি' in str(data)):
# intent='email_CLM_data_q'
# if(probability<0.40):
# if('জি.বি' in str(data) or 'এমবি' in str(data) or 'জিবির' in str(data) or 'এমবি' in str(data) or 'এম্বি' in str(data) or 'জিবি' in str(data)):
# intent='email_CLM_data_q'
f.write(str(data).rstrip()+'|sms|'+intent+'|'+str(probability)+'\n')
f.close()
#parsing = nlu_engine_bn.parse(str(data))
#print(parsing)
#intent = parsing['intent']['intentName']
if(intent in RESPONSE_BN):
return {'text':RESPONSE_BN[intent]}
else:
return {'text':'kono intent paoya jaini [BN]'}
return {'data': 'haha'}
def isascii(s):
"""Check if the characters in string s are in ASCII, U+0-U+7F."""
return len(s) == len(s.encode())
for line in f:
print(line)
words=line.split(' ')
if(len(words)>=5):
response = home(line)
print(response)
print('\n')
time.sleep(1.4)
@ehzawad
Copy link
Author

ehzawad commented Jan 18, 2023 

import pandas as pd
import time
import io
import json
from langdetect import detect
from snips_nlu import SnipsNLUEngine


import time

# with io.open("/home/custom-bot/sms/email_dataset.json") as f:
    # sample_dataset = json.load(f)

with io.open("/home/ehz/Downloads/data_only_bn_new.json") as f:
    sample_dataset_bn = json.load(f)

# nlu_engine = SnipsNLUEngine()
# nlu_engine.fit(sample_dataset)

nlu_engine_bn = SnipsNLUEngine()
nlu_engine_bn.fit(sample_dataset_bn)


RESPONSE_BN = {"SMS_balance_Q": "এস এম এস চেক করে কিভাবে"}
# RESPONSE_EN = {"prepaid_SMS_bundle_q": "Dear Customer, please dial *123*2*7*1# to purchase 100 SMS with 3 days validity at BDT5 (including SD, VAT and SC). To check remaining SMS"}


# creating a Flask app
#app = Flask(__name__)
def isASCII(data):
    try:
        data.encode().decode('ASCII')
    except UnicodeDecodeError:
        return False
    else:
        return True

#unclean_data = pd.read_csv('/home/custom-bot/sms/wow_call_6months.csv')

# print(unclean_data)
# print(unclean_data.columns.tolist())
# print(unclean_data['Subject'])
# super_clean_data = unclean_data[[' Subject',' Customer-Email']].dropna(axis=0, how='any')
#super_clean_data = unclean_data['Customer SMS'].dropna(axis=0, how='any')
# print(super_clean_data)

# super clean data to text
# super_clean_data.to_csv(r'/home/ehz/Downloads/super_clean_data.txt', header=None, index = False, sep='\t', mode='a')
#super_clean_data.to_csv(r'/home/custom-bot/sms/wow_call_6months.txt', header=None, index = False, mode='a')

f = open('/home/ehz/Downloads/test_data.txt', 'r')
# print(f.readlines())

# count = 0
#
# for line in f:
#     count += 1
#     print(base64.b64decode(line))
    # print("{}{}".format(count, line.strip()))



def home(text):
    f = open("/home/ehz/results_SMS_results_Q.csv", "a")
    fl = open("/home/ehz/results_SMS_results_Q.log", "a")
    req=1
    if(req):
        #parsing = nlu_engine.parse("Minute bundle kine kivabe Robi te")
        #data = request.get_json() 

        #data = "hello world"
        #lang = detect(data)
        data = text
        print(str(data))
        lang = detect(data)
        # parsing = nlu_engine.parse(str(data))
        # print(parsing)
        # print(parsing['intent']['intentName'])
        # intent = parsing['intent']['intentName']
        # probability = parsing['intent']['probability']
            

        if(isASCII(data)):
            pass
            # if(intent is None):
                # intent='None'
            # if(probability is None):
                # probability='None'
            
            # if(intent=='None'):
            #     if(data.find('mb')>-1 or data.find('GB')>-1 or data.find('gb')>-1):
            #         intent='email_clm_data_q'
            #         fl.write(str(intent)+'|email_clm_data_q)|'+str(probability)+'|'+str(data)+'\n')
            #         fl.close()
            # if(probability<0.40):
            #     if(data.find('mb')>-1 or data.find('GB')>-1 or data.find('gb')>-1):
            #         intent='email_CLM_data_q'
            #         fl.write(str(intent)+'|email_clm_data_q)|'+str(probability)+'|'+str(data)+'\n')
            #         fl.close()
            # f.write(str(data).rstrip()+'|email|'+intent+'|'+str(probability)+'\n')
            # f.close()
            # if(lang=='en'):
            #     if(intent in RESPONSE_EN):
            #         return {'text':RESPONSE_EN[intent]}
            #     else:
            #         return {'text':'no intent found'}
            # else:
            #     if(intent in RESPONSE_BN):
            #         return {'text':RESPONSE_BN[intent]}
            #     else:
            #         return {'text':'kono intent paoya jaini'}
        else:
            print("Using BN Model.")
            parsing = nlu_engine_bn.parse(str(data))
            print(parsing)
            intent = parsing['intent']['intentName']
            probability = parsing['intent']['probability']
            
            if(intent is None):
                intent='None'
            if(probability is None):
                probability='None'
            
            if (intent == 'None'):
                if ('sms দেখে কি ভাবে' in str(data) or 'এসএমএস দেখে কি ভাবে' in str(data) or 'এস এম এস কি ভাবে ' in str(data) or 'এসএমএস দেখে কি ভাবে' in str(data) or 'SMS চেক' in str(data)):
                    intent = 'SMS_balance_Q'
                    fl.write(str(intent) + '|SMS_balance_Q)|' + str(probability) + '|' + str(data) + '\n')
                    fl.close()  
            # if(intent=='None'):
            #     if('এয়াটেল' in str(data) or 'এয়ারটেলে' in str(data) or 'এয়ারটেল' in str(data)):
            #         intent='email_airtel_related_query_in_robi_helpline'
            #     if('জি.বি' in str(data) or 'এমবি' in str(data) or 'জিবির' in str(data) or 'এমবি' in str(data) or 'এম্বি' in str(data) or 'জিবি' in str(data)):
            #         intent='email_CLM_data_q'
            
            if (probability < 0.40):
                if ('SMS জানতে' in str(data) or 'এসএমএস জানতে' in str(data) or 'এসএমএস চেক' in str(data) or 'এসএমএস' in str(data) or 'SMS চেক' in str(data)):
                    intent = 'SMS_balance_Q'
                    fl.write(str(intent) + '|SMS_balance_Q)|' + str(probability) + '|' + str(data) + '\n')
                    fl.close()
            # if(probability<0.40):
            #     if('জি.বি' in str(data) or 'এমবি' in str(data) or 'জিবির' in str(data) or 'এমবি' in str(data) or 'এম্বি' in str(data) or 'জিবি' in str(data)):
            #         intent='email_CLM_data_q'
            f.write(str(data).rstrip()+'|sms|'+intent+'|'+str(probability)+'\n')
            f.close()
            #parsing = nlu_engine_bn.parse(str(data))
            #print(parsing)
            #intent = parsing['intent']['intentName']
            if(intent in RESPONSE_BN):
                return {'text':RESPONSE_BN[intent]}
            else:
                return {'text':'kono intent paoya jaini [BN]'}
        return {'data': 'haha'}

def isascii(s):
    """Check if the characters in string s are in ASCII, U+0-U+7F."""
    return len(s) == len(s.encode())


for line in f:
    print(line)
    words=line.split(' ')
    if(len(words)>=5):
        response = home(line)
        print(response)
        print('\n')
        time.sleep(1.4)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment