allanbatista

#!/bin/python3

import json
import subprocess

result = subprocess.check_output(['aws', 'logs', 'describe-log-groups']).decode('utf-8')
result = json.loads(result)['logGroups']

for log in result:
    command = f"aws logs put-retention-policy --retention-in-days 7 --log-group-name {log['logGroupName']}"

allanbatista

import json
import requests
from tqdm import tqdm

categories = requests.get("https://api.mercadolibre.com/sites/MLB/categories/all").json()

def get_attributes(cat):
    cat['attributes'] = requests.get(f'https://api.mercadolibre.com/categories/{cat["id"]}/attributes').json()
    
    for subcat in cat.get('children_categories', []):

allanbatista

#!/usr/bin/env bash

CUDA_VERSION=11.1.0
CUDA_NAME=cuda_11.1.0_455.23.05_linux.run

wget https://developer.download.nvidia.com/compute/cuda/${CUDA_VERSION}/local_installers/${CUDA_NAME} && \
chmod +x $CUDA_NAME && \
sudo ./$CUDA_NAME --toolkit --silent --override && \
echo 'echo "# cuda path" >> ~/.bashrc' && \
echo 'export CUDA_PATH=/usr/local/cuda/lib64 >> ~/.bashrc' && \

allanbatista

import os
import s3fs
import math
import torch
import datetime
from sklearn.metrics import f1_score
import numpy as np
import tensorflow as tf
from collections import OrderedDict

allanbatista

allanbatista

CREATE OR REPLACE FUNCTION `project-id.dataset-id.html_to_text`(text STRING) AS (
TRIM(
    REGEXP_REPLACE(
      REGEXP_REPLACE(
        REGEXP_REPLACE(
          REGEXP_REPLACE(
            REGEXP_REPLACE(
              LOWER(text),
              r"<br>|<br\/>|<br\s+\/>|<\/p>", "\n"),
            r"<[^>]*>", ""),

allanbatista

curl -X POST \
  https://visao-v2-b2w-npf.internal.b2w.io/classifications \
  -H 'Content-Type: application/json' \
  -d '{
      "texts": [
          "Liquidificador Mondial Power Branco/Cinza 350W de Potência 2 Velocidades 1,5L",
          "Geladeira / Refrigerador Brastemp Frost Free BRM44 375 Litros - Branca"
      ],
      "images": [
          "/9j/4QAYRXhpZgAASUkqAAgAAAAAAAAAAAAAAP/sABFEdWNreQABAAQAAABLAAD/4QNfaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wLwA8P3hwYWNrZXQgYmVnaW49Iu+7vyIgaWQ9Ilc1TTBNcENlaGlIenJlU3pOVGN6a2M5ZCI/PiA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJBZG9iZSBYTVAgQ29yZSA1LjMtYzAxMSA2Ni4xNDU2NjEsIDIwMTIvMDIvMDYtMTQ6NTY6MjcgICAgICAgICI+IDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+IDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiIHhtbG5zOnhtcE1NPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvbW0vIiB4bWxuczpzdFJlZj0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL3NUeXBlL1Jlc291cmNlUmVmIyIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bXBNTTpPcmlnaW5hbER

allanbatista

import tensorflow as tf
from tensorflow.keras.layers import Layer


class Attention(Layer):
    """
    this is a attention layer compatible with keras and use a tensorflow 2.x functions.
    
    @example
    > from tensorflow.keras.models import Model

allanbatista

#standardSQL
# https://en.it1352.com/article/e546eb6a693c4f8db4c58aa717684d8d.html
CREATE TEMP FUNCTION accent2latin(word STRING) AS
((
  WITH lookups AS (
    SELECT 
    'ç,æ,œ,á,é,í,ó,ú,à,è,ì,ò,ù,ä,ë,ï,ö,ü,ÿ,â,ê,î,ô,û,å,ø,Ø,Å,Á,À,Â,Ä,È,É,Ê,Ë,Í,Î,Ï,Ì,Ò,Ó,Ô,Ö,Ú,Ù,Û,Ü,Ÿ,Ç,Æ,Œ,ñ' AS accents,
    'c,ae,oe,a,e,i,o,u,a,e,i,o,u,a,e,i,o,u,y,a,e,i,o,u,a,o,O,A,A,A,A,A,E,E,E,E,I,I,I,I,O,O,O,O,U,U,U,U,Y,C,AE,OE,n' AS latins
  ),
  pairs AS (

allanbatista

fasttext skipgram \
         -thread $(grep ^cpu\\scores /proc/cpuinfo | uniq |  awk '{print $4}' )\
         -minCount 2 \
         -wordNgrams 1 \
         -minn 2 \
         -maxn 7 \
         -epoch 10 \
         -dim 64 \
         -input train.txt \
         -output $(date +'model-%Y-%m-%dT%H-%M-%S')