Skip to content

Instantly share code, notes, and snippets.

@aliciawilliams
Last active April 23, 2024 22:02
Show Gist options
  • Star 18 You must be signed in to star a gist
  • Fork 11 You must be signed in to fork a gist
  • Save aliciawilliams/2f27bb592d16109c8c977dab5302af13 to your computer and use it in GitHub Desktop.
Save aliciawilliams/2f27bb592d16109c8c977dab5302af13 to your computer and use it in GitHub Desktop.
Entity sentiment analysis on text data in a Google sheet using Cloud Natural Language
/**
* Performs entity sentiment analysis on english text data in a sheet using Cloud Natural Language (cloud.google.com/natural-language/).
*/
var COLUMN_NAME = {
COMMENTS: 'comments',
LANGUAGE: 'language_detected',
TRANSLATION: 'comments_english',
ENTITY: 'entity_sentiment',
ID: 'id'
};
/**
* Creates a ML Tools menu in Google Spreadsheets.
*/
function onOpen() {
SpreadsheetApp.getUi()
.createMenu('ML Tools')
.addItem('Mark Entities and Sentiment', 'markEntitySentiment')
.addToUi();
};
/**
* For each row in the reviewData sheet with a value in "comments" field,
* will run the retrieveEntitySentiment function
* and copy results into the entitySentiment sheet.
*/
function markEntitySentiment() {
// set variables for reviewData sheet
var ss = SpreadsheetApp.getActiveSpreadsheet();
var dataSheet = ss.getSheetByName('reviewData');
var rows = dataSheet.getDataRange();
var numRows = rows.getNumRows();
var values = rows.getValues();
var headerRow = values[0];
// checks to see if entitySentiment sheet is present; if not, creates new sheet and sets header row
var entitySheet = ss.getSheetByName('entitySentiment');
if (entitySheet == null) {
ss.insertSheet('entitySentiment');
var entitySheet = ss.getSheetByName('entitySentiment');
var esHeaderRange = entitySheet.getRange(1,1,1,6);
var esHeader = [['Review ID','Entity','Salience','Sentiment Score','Sentiment Magnitude','Number of mentions']];
esHeaderRange.setValues(esHeader);
};
// find the column index for comments, language_detected, comments_english
var commentsColumnIdx = headerRow.indexOf(COLUMN_NAME.COMMENTS);
var languageColumnIdx = headerRow.indexOf(COLUMN_NAME.LANGUAGE);
var translationColumnIdx = headerRow.indexOf(COLUMN_NAME.TRANSLATION);
var entityColumnIdx = headerRow.indexOf(COLUMN_NAME.ENTITY);
var idColumnIdx = headerRow.indexOf(COLUMN_NAME.ID);
if (entityColumnIdx == -1) {
Browser.msgBox("Error: Could not find the column named " + COLUMN_NAME.ENTITY + ". Please create an empty column with header \"entity_sentiment\" on the reviewData tab.");
return; // bail
};
ss.toast("Analyzing entities and sentiment...");
// Process each row
for (var i = 0; i < numRows; ++i) {
var value = values[i];
var commentEnCellVal = value[translationColumnIdx];
var entityCellVal = value[entityColumnIdx];
var reviewId = value[idColumnIdx];
// Call retrieveEntitySentiment function for each row that has comments and also an empty entity_sentiment cell
if(commentEnCellVal && !entityCellVal) {
var nlData = retrieveEntitySentiment(commentEnCellVal);
// Paste each entity and sentiment score into entitySentiment sheet
var newValues = []
for each (var entity in nlData.entities) {
var row = [reviewId, entity.name, entity.salience, entity.sentiment.score, entity.sentiment.magnitude, entity.mentions.length
];
newValues.push(row);
}
if(newValues.length) {
entitySheet.getRange(entitySheet.getLastRow() + 1, 1, newValues.length, newValues[0].length).setValues(newValues);
}
// Paste "complete" into entity_sentiment column to denote completion of NL API call
dataSheet.getRange(i+1, entityColumnIdx+1).setValue("complete");
}
}
};
/**
* Calls the NL API with a string
* @param {String} line The line of string
* @return {Object} the entities and related sentiment present in the string.
*/
function retrieveEntitySentiment (line) {
var apiKey = "your key here";
var apiEndpoint = 'https://language.googleapis.com/v1/documents:analyzeEntitySentiment?key=' + apiKey;
// Create our json request, w/ text, language, type & encoding
var nlData = {
document: {
language: 'en-us',
type: 'PLAIN_TEXT',
content: line
},
encodingType: 'UTF8'
};
// Package all of the options and the data together for the call
var nlOptions = {
method : 'post',
contentType: 'application/json',
payload : JSON.stringify(nlData)
};
// And make the call
var response = UrlFetchApp.fetch(apiEndpoint, nlOptions);
return JSON.parse(response);
};
@aliciawilliams
Copy link
Author

@MemeRunner Is this an error you are receiving running code in Apps Script? This doesn't look like a column/header error.

@gabrielgz92
Copy link

@aliciawilliams I really enjoyed your work!
There is something I'm struggling with...

While I was submitting some samples I realised that the 'Number of mentions' was always 1, even when I was passing strings repeating a word. According to the API documentation seems right to call entity.mentions.length

So I saved the JSON response for a line that repeats a particular word to my Drive (DriveApp.createFile('test.json', response).

What I've found its that the word repeated it's an entity each time mentioned, it's not being saved as only one entity and different mentions. Here is my JSON response, my code is the same of yours. Do you know what I'm doing wrong?
Thanks

var line = "My family just loved the showers, we were fascinated by the good quality of this showers. Without those showers our experience would be totally different."

{ "entities": [ { "name": "family", "type": "PERSON", "metadata": {}, "salience": 0.5109271, "mentions": [ { "text": { "content": "family", "beginOffset": 3 }, "type": "COMMON", "sentiment": { "magnitude": 0.9, "score": 0.9 } } ], "sentiment": { "magnitude": 0.9, "score": 0.9 } }, { "name": "showers", "type": "OTHER", "metadata": {}, "salience": 0.19720936, "mentions": [ { "text": { "content": "showers", "beginOffset": 25 }, "type": "COMMON", "sentiment": { "magnitude": 0.9, "score": 0.9 } } ], "sentiment": { "magnitude": 0.9, "score": 0.9 } }, { "name": "showers", "type": "OTHER", "metadata": {}, "salience": 0.12079906, "mentions": [ { "text": { "content": "showers", "beginOffset": 81 }, "type": "COMMON", "sentiment": { "magnitude": 0.8, "score": 0.8 } } ], "sentiment": { "magnitude": 0.8, "score": 0.8 } }, { "name": "quality", "type": "OTHER", "metadata": {}, "salience": 0.10624199, "mentions": [ { "text": { "content": "quality", "beginOffset": 65 }, "type": "COMMON", "sentiment": { "magnitude": 0.9, "score": 0.9 } } ], "sentiment": { "magnitude": 0.9, "score": 0.9 } }, { "name": "experience", "type": "OTHER", "metadata": {}, "salience": 0.04473635, "mentions": [ { "text": { "content": "experience", "beginOffset": 116 }, "type": "COMMON", "sentiment": { "magnitude": 0.2, "score": 0.2 } } ], "sentiment": { "magnitude": 0.2, "score": 0.2 } }, { "name": "showers", "type": "OTHER", "metadata": {}, "salience": 0.020086173, "mentions": [ { "text": { "content": "showers", "beginOffset": 104 }, "type": "COMMON", "sentiment": { "magnitude": 0.1, "score": 0.1 } } ], "sentiment": { "magnitude": 0.1, "score": 0.1 } } ], "language": "en-US" }

@aliciawilliams
Copy link
Author

aliciawilliams commented Jun 24, 2019

@gabrielgz92 Thanks! I looked into it and the documentation says that it will consolidate mentions when they are proper nouns (https://cloud.google.com/natural-language/docs/reference/rest/v1/Entity). Now I do see in my data that sometimes entities like "instructions" have been aggregated as mentions in the response data from the API, even though I wouldn't call that a proper noun. Unfortunately, I can't give you an answer as to why!

@gabrielgz92
Copy link

@gabrielgz92 Thanks! I looked into it and the documentation says that it will consolidate mentions when they are proper nouns (https://cloud.google.com/natural-language/docs/reference/rest/v1/Entity). Now I do see in my data that sometimes entities like "instructions" have been aggregated as mentions in the response data from the API, even though I wouldn't call that a proper noun. Unfortunately, I can't give you an answer as to why!

Thanks Alicia! Now I understood, in fact the documentation uses the example with a proper noun.
:)

@MemeRunner
Copy link

MemeRunner commented Nov 21, 2019

I'm trying to follow instructions on this page.
https://cloud.google.com/blog/products/gcp/analyzing-text-in-a-google-sheet-using-cloud-natural-language-api-and-apps-script

However, once I download the reviews.csv and listings.csv file and upload them to Google Drive, I'm then unable to open the reviews.csv file as a Sheet. It just times out with the 4 color circle.

I've tried creating a new Sheet doc, and then Importing it, but I repeatedly get the following error message:
"Sorry, can't import this file right now. Try again later."

I've tried over 10 times with the same result.

I know the file is 27mb, but my internet connection upload speed is 20mbs, which should be sufficient.

If anyone has any suggestions I'd greatly appreciate it.

Thanks
doug

@aliciawilliams
Copy link
Author

@MemeRunner It's true I had the same issue importing reviews.csv. You can copy and paste from the Kaggle website preview pane of the data, or you can use some of these other techniques to filter the data down to one property and then import into Sheets:
https://medium.com/@presactlyalicia/filtering-csv-files-outside-of-a-spreadsheet-c65c5e99f37
https://medium.com/@presactlyalicia/filtering-csv-files-outside-of-a-spreadsheet-f60e75d4c054
https://medium.com/@presactlyalicia/filtering-csv-files-outside-of-a-spreadsheet-3e382ef31c20

@astasicu
Copy link

Same line of code, different issue:
Request failed for [https://language.googleapis.com/v1/documents:analyzeEntitySentiment?key=myAPIkey - redacted] returned code 400. Truncated server response: { "error": { "code": 400, "message": "API key not valid. Please pass a valid API key.", "status": "INVALID_ARGUMENT", "details": ... (use muteHttpExceptions option to examine full response) (line 111, file "Code")
I've set up the NLP API key but I'm not sure why I've made it pitch a fit!

@UncouthScientist Did you enable the NL API in the APIs console? https://cloud.google.com/apis/docs/enable-disable-apis

@aliciawilliams Hi Alicia, I'm getting the same error. I checked and my API is active and I have downloaded the JSON file.
Maybe I got the API Key wrong... isn't it the api key ID?

Thanks in advance...

@aliciawilliams
Copy link
Author

@astasicu Hi! Make sure that you are using an API key- so you should see if it you go to the Cloud Console then Navigation Menu > APIs & Services > Credentials. Your API keys are listed first, and you can copy and paste them into replace "your key here" in the code..keeping the quotation marks.

@astasicu
Copy link

@aliciawilliams Thank you Alicia, I was pasting the wrong key repeatedly (face-palm)... now it works.

@Stellaxu19
Copy link

Does anyone know why only head line appear, without any sentimental scores or magnitude under? How can I find the mistake line? And which platform can help me to debug the code? Waiting for your reply.

@aliciawilliams
Copy link
Author

@Stellaxu19 Please try following the instructions that are now available on the G Suite solutions gallery: https://developers.google.com/gsuite/solutions/feedback-sentiment-analysis

It's the most up-to-date, so might help fix your problem! If not, feel free to followup here.

@aliciawilliams
Copy link
Author

@Stellaxu19 If you open the script editor (Tools menu -> Script Editor) and then run the function directly from there, you can then View -> Execution transcript to see what happened. Sometimes if it's a really short piece of text, the script will successfully execute, but there won't be any entities to report. If there's an error, then you will see more information about the error there as well.

@Stellaxu19
Copy link

@Stellaxu19 If you open the script editor (Tools menu -> Script Editor) and then run the function directly from there, you can then View -> Execution transcript to see what happened. Sometimes if it's a really short piece of text, the script will successfully execute, but there won't be any entities to report. If there's an error, then you will see more information about the error there as well.

Thank you very much. the template works well. when I try to text the retrieveentitysentiment, there is same problem. However, the results can be output.
image

@aliciawilliams
Copy link
Author

aliciawilliams commented Mar 17, 2020

@stellaxu You need to run markEntitySentiment. If you want to run retrieveEntitySentiment on it's own to test, you will need to get rid of the line parameter, and then hard code in a sentence to test on next to content:...something like the below. Then it won't display in the spreadsheet, but you should be able to log the response.

function retrieveEntitySentiment () {
  var apiKey = myApiKey;
  var apiEndpoint = 'https://language.googleapis.com/v1/documents:analyzeEntitySentiment?key=' + apiKey;
  // Create our json request, w/ text, language, type & encoding
  var nlData = {
    document: {
      language: 'en-us',
      type: 'PLAIN_TEXT',
      content: 'I liked the sushi but the location was terrible.'
    },
    encodingType: 'UTF8'
  };
  //  Package all of the options and the data together for the call
  var nlOptions = {
    method : 'post',
    contentType: 'application/json',  
    payload : JSON.stringify(nlData)
  };
  //  And make the call
  var response = UrlFetchApp.fetch(apiEndpoint, nlOptions);
  return JSON.parse(response);
};

@Stellaxu19
Copy link

@aliciawilliams Got it. Thank you very much.

@AlXias
Copy link

AlXias commented Apr 19, 2020

@aliciawilliams I get a Syntax error: Unexpected identifier (line 74, file "Code.gs") Could you please help with this? Thank you.

@aliciawilliams
Copy link
Author

@AlXias It's hard for me to tell from this amount of information. Can you try to recreate your spreadsheet following the instructions on the new solutions page: https://developers.google.com/gsuite/solutions/feedback-sentiment-analysis ? That has the most up-to-date code. Many times, the problem can be with the format of the spreadsheet.

@AlXias
Copy link

AlXias commented Apr 21, 2020

@AlXias It's hard for me to tell from this amount of information. Can you try to recreate your spreadsheet following the instructions on the new solutions page: https://developers.google.com/gsuite/solutions/feedback-sentiment-analysis ? That has the most up-to-date code. Many times, the problem can be with the format of the spreadsheet.

Thanks a lot, @aliciawilliams, it worked! I don't know what the issue was, but I made a copy of the mentioned Google Sheets and modified it. I really appreciate your assistance!

@sdarive
Copy link

sdarive commented Apr 27, 2020

@aliciawilliams I get the same error..Tried serval ways but not sure why is happening..
Syntax error: Unexpected identifier (line 74, file "Code.gs")
It is with this statement for each (var entity in nlData.entities) copied the relevant script below in its entirety..
Pls help and thanks in advance

ss.toast("Analyzing entities and sentiment...");
// Process each row
for (var i = 0; i < numRows; ++i) {
var value = values[i];
var commentEnCellVal = value[translationColumnIdx];
var entityCellVal = value[entityColumnIdx];
var reviewId = value[idColumnIdx];

// Call retrieveEntitySentiment function for each row that has comments and also an empty entity_sentiment cell
if(commentEnCellVal && !entityCellVal) {
    var nlData = retrieveEntitySentiment(commentEnCellVal);
    // Paste each entity and sentiment score into entitySentiment sheet
    var newValues = []
    for each (var entity in nlData.entities) {
      var row = [reviewId, entity.name, entity.salience, entity.sentiment.score, entity.sentiment.magnitude, entity.mentions.length
                ];
      newValues.push(row);
    }
    if(newValues.length) {
      entitySheet.getRange(entitySheet.getLastRow() + 1, 1, newValues.length, newValues[0].length).setValues(newValues);
    }
    // Paste "complete" into entity_sentiment column to denote completion of NL API call
    dataSheet.getRange(i+1, entityColumnIdx+1).setValue("complete");
 }

}
};

@aliciawilliams
Copy link
Author

@sdarive Can you try to follow the same instructions I gave above; to start with the solutions gallery template and just paste in your text data? I'm not sure, but my hypothesis is that the response back from the NL API doesn't identify any entities, so make sure you try it with full sentences. Let me know how it goes.

@sdarive
Copy link

sdarive commented Apr 27, 2020

Thank you @aliciawilliams! I'll try that this evening..I did step through the overall script one step at a time without issues until that line..noticed that the that portion of the script is different than in your earlier versions..

@sdarive
Copy link

sdarive commented Apr 28, 2020

@aliciawilliams..It worked. Thanks!

@sdarive
Copy link

sdarive commented May 1, 2020

@aliciawilliams...figured out the earlier error..It is with the migration of this script execution from earlier Mozilla engine to Chrome V8 engine..I disabled V8 and worked like a charm..guess some syntax doesn't port over well..

@micaviray
Copy link

Hi @aliciawilliams!
I encounter similar error when retrieving Entity Statement:
Exception: Request failed for https://language.googleapis.com returned code 400. Truncated server response: { "error": { "code": 400, "message": "One of content, or gcs_content_uri must be set.", "status": "INVALID_ARGUMENT", "details": ... (use muteHttpExceptions option to examine full response) (line 95, file "Code")

I used the script below:
var docDetails = {
language: 'en-us',
type: 'PLAIN_TEXT',
content: line
};

var nlData = {
document: docDetails,
encodingType: 'UTF8'
};

var nlOptions = {
method: 'post',
contentType: 'application/json',
payload: JSON.stringify(nlData)
};

var response= UrlFetchApp.fetch(apiEndpoint, nlOptions);
Logger.log(response.getContentText());
return JSON.parse(response);
};

Thank you!

@madilk
Copy link

madilk commented Aug 22, 2020

FYI, the retrieveEntitySentiment() function in this gist is correct, but the code sample in the following article is missing the leading quote for the apiEndpoint variable on line 3:

https://cloud.google.com/blog/big-data/2017/12/analyzing-text-in-a-google-sheet-using-cloud-natural-language-api-and-apps-script#closeImage

Thanks!

@madilk
Copy link

madilk commented Aug 22, 2020

@aliciawilliams...figured out the earlier error..It is with the migration of this script execution from earlier Mozilla engine to Chrome V8 engine..I disabled V8 and worked like a charm..guess some syntax doesn't port over well..

Thank you!

@aliciawilliams
Copy link
Author

aliciawilliams commented Sep 11, 2020

@madilk Thanks for the heads up on the blog, I am going to see if I can get it updated. Hopefully I can find some time to get the code updated for the new runtime too.

@madilk
Copy link

madilk commented Oct 26, 2020

@aliciawilliams and others: The max runtime for Google Apps script is 6 min. I was trying NL API on cells where each cell had ~1000 char * 8 K rows. The closest SO thread I found on this topic was here but I'm not sure if/how this would translate to the NL API script posted above. From what I can read [as a non-dev] here, this generic script would check IF run time crosses 5 min THEN save results and start a new call ? https://stackoverflow.com/questions/41971806/how-can-i-increase-the-6-minute-execution-limit-in-google-apps-script

@e-orlov
Copy link

e-orlov commented May 5, 2022

can't be saved as script: error SyntaxError: Unexpected identifier (line 74, file "Code.gs")

image

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment