This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import boto3 | |
import pandas as pd | |
from botocore.exceptions import NoCredentialsError | |
def get_all_guardduty_alerts(max_results_per_detector=50): | |
client = boto3.client('guardduty') | |
all_alerts = [] | |
try: | |
# List all GuardDuty detectors |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import json | |
import pandas as pd | |
from functools import wraps | |
from typing import Any, Callable | |
from pydantic import validate_arguments, BaseModel, validate_call, model_validator, create_model, TypeAdapter | |
import requests, time, pinecone | |
from bs4 import BeautifulSoup | |
from langchain.indexes import VectorstoreIndexCreator | |
from langchain.document_loaders import UnstructuredURLLoader |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
index_name = 'oktaevents' | |
index = pinecone.Index(index_name) | |
embed = OpenAIEmbeddings() | |
embedding = openai.Embedding.create( | |
input=query, | |
model="text-embedding-ada-002" | |
) | |
vector = embedding["data"][0]["embedding"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from langchain.document_loaders import WebBaseLoader | |
from langchain.document_loaders import TextLoader | |
url = 'https://developer.okta.com/docs/reference/api/event-types/#catalog' | |
loader = TextLoader ('/tmp/okta_events.txt') | |
scrape_data = loader.load() | |
index_name = 'oktaevents' | |
# Chunk your data up into smaller documents |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def ask_llm_to_get_filter (items, query): | |
all_text = get_all_items (items) | |
#all_text = get_all_above_score (items, 0.9) | |
role = 'you are an expert at going through text and finding filter expressions inside text given for specific task. Only use the functions you have been provided with.' | |
desc = """ | |
# how to find the filter expresison | |
- the filter expressions have characters with dots inside them. | |
- the task that the filter expression will help with is right around the that filter expression | |
- you must pick one filter expression that best suits the task , the event filter must not have OR or AND |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Your Role | |
You are an excellent research analyst of a topic thats given to you, can do excellent analysis on the topic from all | |
angles and write a comprehensive report on the topic. | |
Your task is to write a comprehensive report to fulfill the objective given to you. | |
# Objective | |
Qualcomm Stock for Medium Term Investment | |
# Your Task |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import sys, os | |
# your paths, below is just an example on mac | |
sys.path .extend ( ['/opt/homebrew/Cellar/python@3.11/3.11.5/Frameworks/Python.framework/Versions/3.11/lib/python311.zip', '/opt/homebrew/Cellar/python@3.11/3.11.5/Frameworks/Python.framework/Versions/3.11/lib/python3.11', '/opt/homebrew/Cellar/python@3.11/3.11.5/Frameworks/Python.framework/Versions/3.11/lib/python3.11/lib-dynload'] ) | |
os.environ["SERPAPI_API_KEY"] = "xxx" | |
from serpapi import GoogleSearch | |
def serp_search (query_json) : |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def visualize_knowledge_graph(kg): | |
print (kg.keys()) | |
dot = Digraph(comment="Knowledge Graph") | |
# Add nodes | |
for node in kg['nodes']: | |
dot.node(str(node['id']), node['label'], color=node['color']) | |
# Add edges | |
for edge in kg['edges']: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
prompt_content = """ | |
Your task is make the knowledge graph from an article for a given objective. | |
objective : {objective} | |
you must follow all reqirements that are listed below | |
## graph requirements | |
- in the edges list, count the number of target ids for each source ids, and the | |
number must be list_target_ids for each node | |
- when you make the edges, ensure that there must not be more than 10 target node | |
ids connected to any given source node id in the graph. the number in list_target_ids | |
for any given source node id must therefore be at most 10 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class Node(BaseModel): | |
""" | |
Node class for the knowledge graph. Each node represents an entity. | |
Attributes: | |
id (int): Unique identifier for the node. | |
label (str): Label or name of the node. | |
color (str): Color of the node. | |
num_targets (int): Number of target nodes connected to this node. |
NewerOlder