Skip to content

Instantly share code, notes, and snippets.

View harazdovskiy's full-sized avatar
🏗️
Building great thing!

Dmytro Harazdovskiy harazdovskiy

🏗️
Building great thing!
View GitHub Profile
@harazdovskiy
harazdovskiy / main.py
Created January 9, 2023 13:30
Python cli script for loading multipart s3 object bigger than 5gb using chunks
import argparse
import boto3
import json
import multiprocessing
# Starts Multipart Upload
def start_upload(bucket, key):
s3_client = boto3.client('s3')
response = s3_client.create_multipart_upload(
{
"took": 22,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
{
"took": 10,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
{
"buckets": [
{
"key_as_string": "2022-12-01T00:00:00.000Z",
"key": 1669852800000,
"doc_count": 146,
"unique": {
"value": 42
}
},
[
{
"key": "Asia",
"doc_count": 1220,
"percentile": {
"values": {
"1.0": 20.984375,
"5.0": 27.984375,
"25.0": 44.96875,
"50.0": 65.25,
{
"buckets": [
{
"key_as_string": "2022-12-01T00:00:00.000Z",
"key": 1669852800000,
"doc_count": 146
},
{
"key_as_string": "2022-12-02T00:00:00.000Z",
"key": 1669939200000,
{
"buckets": [
{
"key_as_string": "2022-11-28T00:00:00.000Z",
"key": 1669593600000,
"doc_count": 582,
"significant_products_categories": {
"doc_count": 582,
"bg_count": 4675,
"buckets": [
const {create} = require("./index.js");
const {client, index} = require("../elastic");
describe('#create', () => {
beforeEach(async () => {
await client.deleteByQuery({
index,
query: {
match_all: {}
}
const {ulid} = require('ulid');
const {client, index} = require("../elastic.js");
module.exports.createHandler = async (request, h) => {
if (Object.keys(request.payload))
try {
const res = await this.create(request.payload)
return h.response(res).code(200);
} catch (e) {
console.log({e})
module.exports = {
preset: '@shelf/jest-elasticsearch',
clearMocks: true,
collectCoverage: true,
coverageDirectory: "coverage",
coverageProvider: "v8"
};