regispires

Comparison of Cloud Object Storage from the most used cloud providers

A brief summary of pricing for Object Storage services from AWS, Azure and GCP providers

AWS - S3

The storage we want is just for backup, so theoretically we won't need access, and even if we do, instant access won't be necessary. So we will use this storage in the cheapest region. In this case: US East (N. Virginia)

Before talking about spending on AWS, we must first raise the reasons why we will be charged. In our case:

regispires

regispires

import java.util.List;
import javax.persistence.Tuple;

import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;

regispires

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import javax.persistence.Tuple;
import javax.persistence.TupleElement;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;

regispires

import os
from google.cloud import firestore
import pandas as pd

os.environ["GOOGLE_APPLICATION_CREDENTIALS"]="myapp-firebase-adminsdk.json"

db = firestore.Client()
users = list(db.collection('usuarios').stream())
users_dict = list(map(lambda x: x.to_dict(), users))
df = pd.DataFrame(users_dict)

regispires

import pandas as pd
from pymongo import MongoClient
import settings

def _connect_mongo(host=settings.MONGO_HOST, port=settings.MONGO_PORT,
        username=settings.MONGO_USER, password=settings.MONGO_PASSWORD, db=settings.MONGO_DB):
    """ A util for making a connection to mongo """
    if username and password:
        mongo_uri = 'mongodb://%s:%s@%s:%s/%s' % (username, password, host, port, db)
        conn = MongoClient(mongo_uri)

regispires

def get_types_dates(df):
    cols_names = list(df)
    cols_types = df.dtypes
    dates = []
    types = {}
    for i in range(len(cols_names)):
        name = cols_names[i]
        type_ = cols_types[i].name
        if type_[:4] == 'date':
            dates.append(name)

regispires

# some parsers that you can use in pandas_import_data_with_dt_check.py
datetime_parser_en_us = lambda x: pd.to_datetime(x, format='%m/%d/%Y %H:%M:%S')
datetime_parser_bt_br = lambda x: pd.to_datetime(x, format='%d/%m/%Y %H:%M:%S')
datetime_parser_global= lambda x: pd.to_datetime(x, format='%Y-%m-%d %H:%M:%S')

date_parser_en_us = lambda x: pd.to_datetime(x, format='%m/%d/%Y')
date_parser_pt_br = lambda x: pd.to_datetime(x, format='%d/%m/%Y')
date_parser_global= lambda x: pd.to_datetime(x, format='%Y-%m-%d')

regispires

import sys
import numpy as np
import pandas as pd

def sizeof_fmt(num, suffix='B'):
    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
        if abs(num) < 1024.0:
            return "%3.1f %s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f %s%s" % (num, 'Yi', suffix)

regispires

# for security reasons this file must be added to your .gitignore file
MONGO_HOST=<my_host_name_or_address>
MONGO_PORT=27017
MONGO_DB=<my_database>
MONGO_USER=<my_user>
MONGO_PASSWORD=<my_password>