Skip to content

Instantly share code, notes, and snippets.

@briandenicola

briandenicola/search.config.cs

Last active September 28, 2021 22:03
Show Gist options
  • Save briandenicola/301051487f64e64fc1aa7cb4df828b1b to your computer and use it in GitHub Desktop.
Save briandenicola/301051487f64e64fc1aa7cb4df828b1b to your computer and use it in GitHub Desktop.
Download ZIP
Azure Search Code Samples
Raw
search.config.cs
using System;
using Azure;
using System.Collections.Generic;
using System.Net;
using System.Threading.Tasks;
using Microsoft.Azure.Search;
using Microsoft.Azure.Search.Models;
using Azure.Search.Documents;
using Azure.Search.Documents.Indexes;
using Azure.Search.Documents.Indexes.Models;
using Azure.Search.Documents.Models;
using Microsoft.Rest.Azure;
namespace searchclient
{
class Program
{
static void Main(string[] args)
{
string serviceName = "qnaeg1-asid34wdoliepwc";
string indexName = "margieswebsitedocs";
string apiKey = Environment.GetEnvironmentVariable("api-key");
// Create a SearchIndexClient to send create/delete index commands
Uri serviceEndpoint = new Uri($"https://{serviceName}.search.windows.net/");
AzureKeyCredential credential = new AzureKeyCredential(apiKey);
var idxclient = new Azure.Search.Documents.Indexes.SearchIndexClient(serviceEndpoint, credential);
// Create a SearchClient to load and query documents
var qryclient = new Azure.Search.Documents.SearchClient(serviceEndpoint, indexName, credential);
var index = new SearchIndex(indexName)
{
Fields =
{
new SimpleField("DocumentID", SearchFieldDataType.String) { IsKey = true, IsFilterable = true, IsSortable = true },
new SearchableField("DocumentURL") { IsFilterable = true, IsSortable = true },
new SearchableField("FileName") { IsFilterable = true, IsSortable = true },
new SearchableField("FileContent") { IsFilterable = true, IsSortable = true },
new SimpleField("DocumentSize", SearchFieldDataType.Int64) { IsFilterable = true, IsSortable = true },
new SearchableField("LastModified") { IsFilterable = true, IsSortable = true },
new SimpleField("SentimentScore", SearchFieldDataType.Double) { IsFilterable = true, IsSortable = true },
new SimpleField("KeyPhrases", SearchFieldDataType.Collection(SearchFieldDataType.String)) { IsFilterable = true },
new SimpleField("Urls", SearchFieldDataType.Collection(SearchFieldDataType.String)) { IsFilterable = true },
new SimpleField("Persons", SearchFieldDataType.Collection(SearchFieldDataType.String)) { IsFilterable = true },
new SimpleField("Locations", SearchFieldDataType.Collection(SearchFieldDataType.String)) { IsFilterable = true },
new SimpleField("OCRText", SearchFieldDataType.Collection(SearchFieldDataType.String)) { IsFilterable = true },
new SimpleField("TopTenWords", SearchFieldDataType.Collection(SearchFieldDataType.String)) { IsFilterable = true },
new ComplexField("description", true)
{
Fields =
{
new SimpleField("tags", SearchFieldDataType.Collection(SearchFieldDataType.String)) { IsFilterable = true },
new ComplexField("captions", true)
{
Fields =
{
new SimpleField("text", SearchFieldDataType.String) { IsFilterable = true },
new SimpleField("confidence", SearchFieldDataType.Double) { IsFilterable = true }
}
}
}
}
}
Console.WriteLine("{0}", "Creating index...\n");
idxclient.DeleteIndex(index);
idxclient.CreateIndex(index);
var blobDataSource = DataSource.AzureBlobStorage(
name: "margieswebsitedocs",
storageConnectionString: Environment.GetEnvironmentVariable("STORAGE_CON_STR"),
containerName: "websitedocs");
Console.WriteLine("{0}", "Creating data source...\n");
var searchService = new SearchServiceClient(serviceName, new SearchCredentials(apiKey));
searchService.DataSources.CreateOrUpdate(blobDataSource);
Console.WriteLine("Creating Blob Storage indexer...\n");
var map = new List<Microsoft.Azure.Search.Models.FieldMapping> {
new Microsoft.Azure.Search.Models.FieldMapping("metadata_storage_name", "DocumentID", Microsoft.Azure.Search.Models.FieldMappingFunction.Base64Encode()),
new Microsoft.Azure.Search.Models.FieldMapping("metadata_storage_path", "DocumentURL"),
new Microsoft.Azure.Search.Models.FieldMapping("metadata_storage_name", "FileName"),
new Microsoft.Azure.Search.Models.FieldMapping("content", "FileContent"),
new Microsoft.Azure.Search.Models.FieldMapping("metadata_storage_size", "DocumentSize"),
new Microsoft.Azure.Search.Models.FieldMapping("metadata_storage_last_modified", "LastModified")
};
var outputMap = new List<Microsoft.Azure.Search.Models.FieldMapping> {
new Microsoft.Azure.Search.Models.FieldMapping("/document/mySentiment", "SentimentScore"),
new Microsoft.Azure.Search.Models.FieldMapping("/document/keyPhrases", "KeyPhrases"),
new Microsoft.Azure.Search.Models.FieldMapping("/document/urls", "Urls"),
new Microsoft.Azure.Search.Models.FieldMapping("/document/persons", "Persons"),
new Microsoft.Azure.Search.Models.FieldMapping("/document/locations", "Locations"),
new Microsoft.Azure.Search.Models.FieldMapping("/document/normalized_images/*/description", "description"),
new Microsoft.Azure.Search.Models.FieldMapping("/document/normalized_images/*/text", "OCRText"),
new Microsoft.Azure.Search.Models.FieldMapping("/document/myWordList", "TopTenWords")
};
var p = new Microsoft.Azure.Search.Models.IndexingParameters();
var options = new Dictionary<string, object>();
options.Add("allowSkillsetToReadFileData", true);
p.Configuration = options;
Indexer blobIndexer = new Indexer(
name: "blob-indexer",
dataSourceName: blobDataSource.Name,
skillsetName: "vacationskills",
targetIndexName: indexName,
outputFieldMappings: outputMap,
fieldMappings: map,
parameters: p,
schedule: new Microsoft.Azure.Search.Models.IndexingSchedule(TimeSpan.FromDays(1)));
bool exists = searchService.Indexers.Exists(blobIndexer.Name);
if (exists)
{
searchService.Indexers.Reset(blobIndexer.Name);
}
searchService.Indexers.CreateOrUpdate(blobIndexer);
Console.WriteLine("Running Blob Storage indexer...\n");
try
{
searchService.Indexers.Run(blobIndexer.Name);
}
catch (CloudException e) when (e.Response.StatusCode == (HttpStatusCode)429)
{
Console.WriteLine("Failed to run indexer: {0}", e.Response.Content);
}
}
}
}
Raw
search.index.json
{
"name": "margieswebsitedocs",
"fields": [
{
"name": "DocumentID",
"type": "Edm.String",
"facetable": false,
"filterable": true,
"key": true,
"retrievable": true,
"searchable": false,
"sortable": true,
"analyzer": null,
"indexAnalyzer": null,
"searchAnalyzer": null,
"synonymMaps": [],
"fields": []
},
{
"name": "DocumentURL",
"type": "Edm.String",
"facetable": false,
"filterable": true,
"key": false,
"retrievable": true,
"searchable": true,
"sortable": true,
"analyzer": null,
"indexAnalyzer": null,
"searchAnalyzer": null,
"synonymMaps": [],
"fields": []
},
{
"name": "FileName",
"type": "Edm.String",
"facetable": false,
"filterable": true,
"key": false,
"retrievable": true,
"searchable": true,
"sortable": true,
"analyzer": null,
"indexAnalyzer": null,
"searchAnalyzer": null,
"synonymMaps": [],
"fields": []
},
{
"name": "FileContent",
"type": "Edm.String",
"facetable": false,
"filterable": true,
"key": false,
"retrievable": true,
"searchable": true,
"sortable": true,
"analyzer": null,
"indexAnalyzer": null,
"searchAnalyzer": null,
"synonymMaps": [],
"fields": []
},
{
"name": "DocumentSize",
"type": "Edm.Int64",
"facetable": false,
"filterable": true,
"retrievable": true,
"sortable": true,
"analyzer": null,
"indexAnalyzer": null,
"searchAnalyzer": null,
"synonymMaps": [],
"fields": []
},
{
"name": "LastModified",
"type": "Edm.DateTimeOffset",
"facetable": false,
"filterable": true,
"retrievable": true,
"sortable": true,
"analyzer": null,
"indexAnalyzer": null,
"searchAnalyzer": null,
"synonymMaps": [],
"fields": []
},
{
"name": "SentimentScore",
"type": "Edm.Double",
"facetable": false,
"filterable": true,
"retrievable": true,
"sortable": true,
"analyzer": null,
"indexAnalyzer": null,
"searchAnalyzer": null,
"synonymMaps": [],
"fields": []
},
{
"name": "KeyPhrases",
"type": "Collection(Edm.String)",
"facetable": false,
"filterable": true,
"retrievable": true,
"searchable": true,
"analyzer": null,
"indexAnalyzer": null,
"searchAnalyzer": null,
"synonymMaps": [],
"fields": []
},
{
"name": "Urls",
"type": "Collection(Edm.String)",
"facetable": false,
"filterable": true,
"retrievable": true,
"searchable": false,
"analyzer": null,
"indexAnalyzer": null,
"searchAnalyzer": null,
"synonymMaps": [],
"fields": []
},
{
"name": "Persons",
"type": "Collection(Edm.String)",
"facetable": false,
"filterable": true,
"retrievable": true,
"searchable": false,
"analyzer": null,
"indexAnalyzer": null,
"searchAnalyzer": null,
"synonymMaps": [],
"fields": []
},
{
"name": "Locations",
"type": "Collection(Edm.String)",
"facetable": false,
"filterable": true,
"retrievable": true,
"searchable": true,
"analyzer": "en.microsoft",
"indexAnalyzer": null,
"searchAnalyzer": null,
"synonymMaps": [
"mysynonymmap"
],
"fields": []
},
{
"name": "description",
"type": "Collection(Edm.ComplexType)",
"analyzer": null,
"synonymMaps": [],
"fields": [
{
"name": "tags",
"type": "Collection(Edm.String)",
"facetable": false,
"filterable": true,
"retrievable": true,
"searchable": true,
"analyzer": "standard.lucene",
"indexAnalyzer": null,
"searchAnalyzer": null,
"synonymMaps": [],
"fields": []
},
{
"name": "captions",
"type": "Collection(Edm.ComplexType)",
"analyzer": null,
"synonymMaps": [],
"fields": [
{
"name": "text",
"type": "Edm.String",
"facetable": false,
"filterable": true,
"key": false,
"retrievable": true,
"searchable": true,
"sortable": false,
"analyzer": "standard.lucene",
"indexAnalyzer": null,
"searchAnalyzer": null,
"synonymMaps": [],
"fields": []
},
{
"name": "confidence",
"type": "Edm.Double",
"facetable": false,
"filterable": true,
"retrievable": true,
"sortable": false,
"analyzer": null,
"indexAnalyzer": null,
"searchAnalyzer": null,
"synonymMaps": [],
"fields": []
}
]
}
]
},
{
"name": "OCRText",
"type": "Collection(Edm.String)",
"facetable": true,
"filterable": true,
"retrievable": true,
"searchable": true,
"analyzer": "standard.lucene",
"indexAnalyzer": null,
"searchAnalyzer": null,
"synonymMaps": [],
"fields": []
},
{
"name": "TopTenWords",
"type": "Collection(Edm.String)",
"facetable": true,
"filterable": true,
"retrievable": true,
"searchable": true,
"analyzer": "standard.lucene",
"indexAnalyzer": null,
"searchAnalyzer": null,
"synonymMaps": [],
"fields": []
}
],
"suggesters": [
{
"name": "sg",
"searchMode": "analyzingInfixMatching",
"sourceFields": [
"Locations"
]
}
],
"scoringProfiles": [
{
"name": "scoring",
"functions": [
{
"fieldName": "DocumentSize",
"freshness": null,
"interpolation": "linear",
"magnitude": {
"boostingRangeStart": 1,
"boostingRangeEnd": 5,
"constantBoostBeyondRange": false
},
"distance": null,
"tag": null,
"type": "magnitude",
"boost": 10
},
{
"fieldName": "LastModified",
"freshness": {
"boostingDuration": "P365D"
},
"interpolation": "linear",
"magnitude": null,
"distance": null,
"tag": null,
"type": "freshness",
"boost": 10
}
],
"functionAggregation": 0,
"text": {
"weights": {
"KeyPhrases": 10
}
}
},
{
"name": "fake",
"functions": [],
"functionAggregation": null
}
],
"defaultScoringProfile": "scoring",
"corsOptions": null,
"analyzers": [],
"charFilters": [],
"tokenFilters": [],
"tokenizers": [],
"@odata.etag": "\"0x8D85B4376A91136\""
}
Raw
search.indexer.json
{
"@odata.context": "https://qnaeg1-asid34wdoliepwc.search.windows.net/$metadata#indexers/$entity",
"@odata.etag": "\"0x8D85B47231EB22D\"",
"name": "blob-indexer",
"description": null,
"dataSourceName": "margieswebsitedocs",
"skillsetName": "vacationskills",
"targetIndexName": "margieswebsitedocs",
"disabled": null,
"schedule": {
"interval": "P1D",
"startTime": "2020-09-16T17:38:28.038Z"
},
"parameters": {
"batchSize": null,
"maxFailedItems": null,
"maxFailedItemsPerBatch": null,
"base64EncodeKeys": null,
"configuration": {
"allowSkillsetToReadFileData": true,
"dataToExtract": "contentAndMetadata",
"imageAction": "generateNormalizedImages"
}
},
"fieldMappings": [
{
"sourceFieldName": "metadata_storage_name",
"targetFieldName": "DocumentID",
"mappingFunction": {
"name": "base64Encode"
}
},
{
"sourceFieldName": "metadata_storage_path",
"targetFieldName": "DocumentURL"
},
{
"sourceFieldName": "metadata_storage_name",
"targetFieldName": "FileName"
},
{
"sourceFieldName": "content",
"targetFieldName": "FileContent"
},
{
"sourceFieldName": "metadata_storage_size",
"targetFieldName": "DocumentSize"
},
{
"sourceFieldName": "metadata_storage_last_modified",
"targetFieldName": "LastModified"
}
],
"outputFieldMappings": [
{
"sourceFieldName": "/document/mySentiment",
"targetFieldName": "SentimentScore"
},
{
"sourceFieldName": "/document/keyPhrases",
"targetFieldName": "KeyPhrases"
},
{
"sourceFieldName": "/document/urls",
"targetFieldName": "Urls"
},
{
"sourceFieldName": "/document/persons",
"targetFieldName": "Persons"
},
{
"sourceFieldName": "/document/locations",
"targetFieldName": "Locations"
},
{
"sourceFieldName": "/document/normalized_images/*/description",
"targetFieldName": "description"
},
{
"sourceFieldName": "/document/normalized_images/*/text",
"targetFieldName": "OCRText"
},
{
"sourceFieldName": "/document/myWordList",
"targetFieldName": "TopTenWords"
}
],
"cache": null,
"encryptionKey": null
}
Raw
search.queries.txt
Challenge #1
https://qnaeg1.azurewebsites.net/qnamaker/knowledgebases/4f1e1b05-a7d4-4573-b34b-7597384469c1/generateAnswer
Challenge #2
Query All
https://qnaeg1-asid34wdoliepwc.search.windows.net/indexes/margieswebsitedocs/docs?api-version=2020-06-30&search=*
Challenge #3
Hotels with positive reviews
https://qnaeg1-asid34wdoliepwc.search.windows.net/indexes/margieswebsitedocs/docs?api-version=2020-06-30&search=&$filter=SentimentScore lt 0.5
Location of Las Vegas
https://qnaeg1-asid34wdoliepwc.search.windows.net/indexes/margieswebsitedocs/docs?api-version=2019-05-06&search=*&$filter=Locations/any(l: l eq 'Las Vegas')&$select=FileName,Locations
NY Reviews
https://qnaeg1-asid34wdoliepwc.search.windows.net/indexes/margieswebsitedocs/docs?api-version=2020-06-30&$count=true&$searchMode=All&$search=DocumentURL:("review")&$select=DocumentURL,SentimentScore,KeyPhrases,Locations&$filter=Locations/any(l: l eq 'New York')
Negative LV Reviews
https://qnaeg1-asid34wdoliepwc.search.windows.net/indexes/margieswebsitedocs/docs?api-version=2020-06-30&$count=true&$searchMode=All&$search=DocumentURL:("review")&$select=DocumentURL,SentimentScore,KeyPhrases,Locations&$filter=Locations/any(l: l eq 'Las Vegas') and SentimentScore lt 0.51
Broadway Reviews
https://qnaeg1-asid34wdoliepwc.search.windows.net/indexes/margieswebsitedocs/docs?api-version=2020-06-30&$count=true&$searchMode=All&$search=DocumentURL:("review")&$select=DocumentURL,SentimentScore,KeyPhrases,Locations&$filter=KeyPhrases/any(k: k eq 'Broadway') and Locations/any(c: c eq 'New York City')
Challenge #4
Grand Canyon
https://qnaeg1-asid34wdoliepwc.search.windows.net/indexes/margieswebsitedocs/docs?api-version=2020-06-30&search=DocumentURL:("collateral")&$count=true&$searchMode=All&$filter=KeyPhrases/any(k: k eq 'Grand Canyon')
Challenge #5
TopTen Keyword
https://qnaeg1-asid34wdoliepwc.search.windows.net/indexes/margieswebsitedocs/docs?api-version=2020-06-30&search=*&$count=true&$searchMode=All&$filter=TopTenWords/any(t: t eq 'dubai')
Challenge #6
Synonyms
https://qnaeg1-asid34wdoliepwc.search.windows.net/indexes/margieswebsitedocs/docs?api-version=2020-06-30&search=USA&searchFields=Locations&%24count=true&%24select=Locations
Suggestor
https://qnaeg1-asid34wdoliepwc.search.windows.net/indexes/margieswebsitedocs/docs/suggest?api-version=2020-06-30&search=UA&suggesterName=sg
Raw
search.skills.pipline.json
{
"@odata.context": "https://qnaeg1-asid34wdoliepwc.search.windows.net/$metadata#skillsets/$entity",
"@odata.etag": "\"0x8D85B47177C7C9F\"",
"name": "vacationskills",
"description": "Extract entities, detect language and extract key-phrases",
"skills": [
{
"@odata.type": "#Microsoft.Skills.Vision.ImageAnalysisSkill",
"name": "#1",
"description": "Extract image analysis.",
"context": "/document/normalized_images/*",
"defaultLanguageCode": "en",
"visualFeatures": [
"description"
],
"details": [],
"inputs": [
{
"name": "image",
"source": "/document/normalized_images/*"
}
],
"outputs": [
{
"name": "description",
"targetName": "description"
}
]
},
{
"@odata.type": "#Microsoft.Skills.Vision.OcrSkill",
"name": "#2",
"description": "Extract text (plain and structured) from image.",
"context": "/document/normalized_images/*",
"textExtractionAlgorithm": null,
"lineEnding": "Space",
"defaultLanguageCode": "en",
"detectOrientation": true,
"inputs": [
{
"name": "image",
"source": "/document/normalized_images/*"
}
],
"outputs": [
{
"name": "text",
"targetName": "text"
}
]
},
{
"@odata.type": "#Microsoft.Skills.Text.MergeSkill",
"name": "#3",
"description": "Create merged_text, which includes all the textual representation of each image inserted at the right location in the content field.",
"context": "/document",
"insertPreTag": " ",
"insertPostTag": " ",
"inputs": [
{
"name": "text",
"source": "/document/content"
},
{
"name": "itemsToInsert",
"source": "/document/normalized_images/*/text"
},
{
"name": "offsets",
"source": "/document/normalized_images/*/contentOffset"
}
],
"outputs": [
{
"name": "mergedText",
"targetName": "merged_text"
}
]
},
{
"@odata.type": "#Microsoft.Skills.Custom.WebApiSkill",
"name": "#5",
"description": null,
"context": "/document",
"uri": "https://tlcchall5.azurewebsites.net/api/funcchall5?code=iPO2ikUKaqcHjcMRZBlVPty9wEuQkeec1l1fKKw4/VRDNFUABpcf4Q==",
"httpMethod": "POST",
"timeout": "PT30S",
"batchSize": 1000,
"degreeOfParallelism": null,
"inputs": [
{
"name": "Text",
"source": "/document/merged_text"
}
],
"outputs": [
{
"name": "wordList",
"targetName": "myWordList"
}
],
"httpHeaders": {}
},
{
"@odata.type": "#Microsoft.Skills.Text.EntityRecognitionSkill",
"name": "#6",
"description": null,
"context": "/document",
"categories": [
"Person",
"URL",
"Location"
],
"defaultLanguageCode": "en",
"minimumPrecision": null,
"includeTypelessEntities": null,
"inputs": [
{
"name": "text",
"source": "/document/merged_text"
}
],
"outputs": [
{
"name": "persons",
"targetName": "persons"
},
{
"name": "urls",
"targetName": "urls"
},
{
"name": "locations",
"targetName": "locations"
}
]
},
{
"@odata.type": "#Microsoft.Skills.Text.SentimentSkill",
"name": "#7",
"description": null,
"context": "/document",
"defaultLanguageCode": "en",
"inputs": [
{
"name": "text",
"source": "/document/merged_text"
}
],
"outputs": [
{
"name": "score",
"targetName": "mySentiment"
}
]
},
{
"@odata.type": "#Microsoft.Skills.Text.KeyPhraseExtractionSkill",
"name": "#8",
"description": null,
"context": "/document",
"defaultLanguageCode": "en",
"maxKeyPhraseCount": null,
"inputs": [
{
"name": "text",
"source": "/document/merged_text"
}
],
"outputs": [
{
"name": "keyPhrases",
"targetName": "keyPhrases"
}
]
}
],
"cognitiveServices": {
"@odata.type": "#Microsoft.Azure.Search.CognitiveServicesByKey",
"description": ""
"key": ""
},
"knowledgeStore": null
"encryptionKey": null
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment