This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from typing import Any | |
class Action: | |
def __init__(self, actionname: str = None): | |
if actionname is not None: | |
self.actionname = actionname | |
else: | |
self.actionname = "UnnamedAction" | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import dlt | |
from typing import Any | |
from dlt.extract import DltResource | |
from dlt.destinations import filesystem | |
from core.actions import Action | |
class WriteFile(Action): |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from typing import Any, List | |
from core.actions import Action | |
class GithubIssueToChat(Action): | |
def __init__(self, actionname: str = None): | |
super().__init__(actionname) | |
def do(self, issues:List, comments:List, *args:Any, **kwargs: Any) -> List: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from core.readers.dlt import Read | |
import dlt | |
from dlt.sources.helpers import requests | |
from typing import Any, Dict, Optional, List | |
class ReadGithub(Read): | |
def __init__(self, actionname:str = None): | |
super().__init__(actionname) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import dlt | |
from dlt.common.typing import TDataItems | |
from dlt.common.schema import TTableSchema | |
from dlt.common.destination import Destination | |
from typing import Any | |
from core.actions import Action | |
class Read(Action): | |
def __init__(self, actionname: str = None): |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from transformers import AutoTokenizer, AutoModelForCausalLM | |
import torch | |
from typing import Any | |
from core.actions import Action | |
class InferWithLlama3Instruct(Action): | |
def __init__( | |
self, | |
hf_token: str, |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from transformers import pipeline as hf_pipeline | |
from typing import Any | |
from core.actions import Action | |
class InferWithHuggingface(Action): | |
def __init__( | |
self, | |
actionname: str = None, |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from core.inference.huggingface import InferWithHuggingface | |
from core.readers.github import ReadGithubIssue | |
from core.inference.llama3instruct import InferWithLlama3Instruct | |
from core.transforms.github import GithubIssueToChat | |
from core.writers.dlt import WriteFile | |
import os | |
# Get the Huggingface token | |
HF_TOKEN = os.getenv("HF_TOKEN") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
INSERT INTO opinions.stattopic (SnapshotDateId, Topic, Tags, TagCount, cntOrigPublishers, cntRepostWRs, | |
cntPositives, cntNegatives, cntAmbiguous, cntGeneral, SentimentHashes, OrigWebResourceHashes, RepostWebResourceHashes ) | |
WITH | |
p AS ( | |
SELECT 20170630 AS SnapshotDateId | |
), | |
SentimentTags AS ( | |
SELECT p.SnapshotDateId, s.SentimentHash, t.Tag, t.GoodAsTopic, s.Tags AS Tags | |
FROM p, opinions.sentiment s, UNNEST(s.Tags) AS t | |
WHERE |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
INSERT INTO opinions.stattopic (SnapshotDateId, Topic, Tags, TagCount, cntOrigPublishers, cntRepostWRs, | |
cntPositives, cntNegatives, cntAmbiguous, cntGeneral, SentimentHashes, OrigWebResourceHashes, RepostWebResourceHashes ) | |
WITH | |
p AS ( | |
SELECT 20170630 AS SnapshotDateId | |
), | |
CalcStatSentiments AS ( | |
SELECT p.SnapshotDateId, t.Tag, t.GoodAsTopic, d.DocumentHash AS DocumentHash, s.SentimentHash, | |
wrOrig.WebResourceHash AS OrigWebResourceHash, wrOrig.Domain AS OrigDomain, wrRepost.WebResourceHash AS RepostWebResourceHash, | |
s.DominantValence AS Valence, d.PublicationTime AS PublicationTime |
NewerOlder