juanroldanbrz

requests~=2.26.0
telegram
aioschedule==0.5.2
bs4==0.0.1
python-telegram-bot==13.1
redis==3.5.3
telebot==0.0.4
Telethon==1.19.0
selenium~=3.141.0

juanroldanbrz

from datetime import datetime, timedelta

from airflow import DAG
from airflow.operators import DummyOperator, PythonOperator

import patents_crawler

default_args = {
    'owner': 'juan.roldan@bluggie.com',
    'start_date': datetime(2019, 1, 1),

juanroldanbrz

from typing import List

from model import PatentDto


def export_patents(filename: str, patents: List[PatentDto]):
    """
    Export patent DTOs to CSV files.
    :param filename: output filename.
    :param patents: list of patents to serialize.

juanroldanbrz

import logging
import os

import boto3
from botocore.exceptions import NoCredentialsError

ACCESS_KEY = os.getenv('AWS_ACCESS_KEY')
SECRET_KEY = os.getenv('AWS_SECRET_KEY')
AWS_BUCKET_NAME = os.getenv('AWS_BUCKET_NAME')
AWS_BUCKET_REGION = os.getenv('AWS_S3_REGION')

juanroldanbrz

class PatentDto:
    """
    Encapsulate patents information.
    """

    def __init__(self, patent_id: str, patent_nr: str, patent_title: str, patent_date: str, keyword: str):
        self.patent_id = patent_id
        self.patent_nr = patent_nr
        self.patent_title = patent_title
        self.patent_date = patent_date

juanroldanbrz

import logging
from datetime import datetime
from typing import List

import requests

import csv_exporter
import s3
from model import PatentDto

juanroldanbrz

version: '3.4'

services:
  airflow:
    image: puckel/docker-airflow:latest
    ports:
      - "8090:8080"
    environment:
      AWS_ACCESS_KEY: xxx
      AWS_SECRET_KEY: xxxx

juanroldanbrz

version: "2.1"
services:
  mongo:
    image: mongo:3.6
    ports:
    - 27017:27017
    command: --smallfiles
    volumes:
      - mongo:/data/db

juanroldanbrz

import org.junit.Test;

import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;


public class Ramon {

juanroldanbrz

   <!-- For testing -->
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-test</artifactId>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.spockframework</groupId>
            <artifactId>spock-core</artifactId>
            <version>${spock.version}</version>