Skip to content

Instantly share code, notes, and snippets.

View sergio11's full-sized avatar
🏠
Working from home

Sergio Sánchez Sánchez sergio11

🏠
Working from home
View GitHub Profile
@sergio11
sergio11 / natural_language_proccessing_operator.py
Created December 10, 2023 20:01
natural_language_proccessing_operator.py
from collections import Counter
from bson import ObjectId
from operators.base_custom_operator import BaseCustomOperator
from airflow.utils.decorators import apply_defaults
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
import numpy as np
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
class NaturalLanguageProccessingOperator(BaseCustomOperator):
"""
@sergio11
sergio11 / index_to_elasticsearch_operator.py
Created December 10, 2023 19:54
index_to_elasticsearch_operator.py
from elasticsearch import Elasticsearch
from airflow.utils.decorators import apply_defaults
from operators.base_custom_operator import BaseCustomOperator
from bson import ObjectId
from datetime import datetime
class IndexToElasticsearchOperator(BaseCustomOperator):
"""
Custom operator to index meeting information to Elasticsearch and update MongoDB.
@sergio11
sergio11 / translation_operator.py
Created December 10, 2023 19:52
translation_operator.py
from bson import ObjectId
from operators.base_custom_operator import BaseCustomOperator
from googletrans import Translator
from airflow.utils.decorators import apply_defaults
class TranslationOperator(BaseCustomOperator):
"""
Custom Airflow Operator for translating transcribed text into multiple target languages.
This operator translates transcribed text into different target languages using the Google Translate API
@sergio11
sergio11 / generate_summary_operator.py
Created December 10, 2023 19:48
generate_summary_operator.py
from bson import ObjectId
from operators.base_custom_operator import BaseCustomOperator
from airflow.utils.decorators import apply_defaults
from transformers import pipeline
class GenerateSummaryOperator(BaseCustomOperator):
"""
Operator responsible for generating a summary based on meeting transcribed text
using a summarization model and storing it in the BSON document in MongoDB.
@sergio11
sergio11 / transcription_operator.py
Created December 10, 2023 19:41
transcription_operator.py
import tempfile
from bson import ObjectId
from operators.base_custom_operator import BaseCustomOperator
from airflow.utils.decorators import apply_defaults
from moviepy.editor import AudioFileClip
import speech_recognition as sr
class TranscriptionOperator(BaseCustomOperator):
"""
An operator that performs audio transcription for meetings.
@sergio11
sergio11 / real_time_streaming_api.py
Created December 2, 2023 12:17
Real-time Streaming API
from client_manager import ClientManager
from flask import Flask, request
from flask_socketio import SocketIO, emit
import os
import logging
from pymongo import MongoClient
from bson import ObjectId
import jwt
# Event logging configuration
@sergio11
sergio11 / secure_provisioning_of_fog_node_access.py
Created December 2, 2023 12:09
Secure Provisioning of Fog Node Access
import datetime
import logging
from common.helpers import generate_response
from flask import Flask, request
import hashlib
import uuid
import os
import redis
from bson import ObjectId
from pymongo import MongoClient
@sergio11
sergio11 / camera_management_services.py
Created December 2, 2023 12:07
Camera Management Services
import base64
import io
import logging
from flask import Flask, request, send_file
from pymongo import MongoClient
import os
from datetime import datetime
from common.requires_authentication_decorator import requires_authentication
from minio import Minio
from common.helpers import generate_response
@sergio11
sergio11 / flask_services_for_user_authentication.py
Created December 2, 2023 12:05
Flask Services for User Authentication
import logging
from common.helpers import generate_response
from common.requires_authentication_decorator import requires_authentication
from flask import Flask, request
import hashlib
import os
import jwt
from pymongo import MongoClient, DESCENDING
@sergio11
sergio11 / video_frame_processor.py
Created December 2, 2023 12:02
Video Frame Processor
from environment import KAFKA_BOOTSTRAP_SERVERS, KAFKA_INPUT_TOPIC, KAFKA_GROUP_ID, KAFKA_OUTPUT_TOPIC
from kafka_connectivity_check import kafka_connectivity_check
from get_flink_environment import get_flink_environment
from create_source_table import create_source_table
from create_sink_table import create_sink_table
from pyflink.table.expressions import col, call
from pyflink.table import DataTypes, Row
from pyflink.table.udf import udtf, TableFunction
from logger import logger
from pyflink.table import Row