Skip to content

Instantly share code, notes, and snippets.

@ilantoren
ilantoren / reshape_query.json
Created May 20, 2022 13:58
Query to reshape GDELT DATA - add fields
[{$addFields: {
actorCodes: {
$concatArrays: [
[
'$Actor1Code'
],
[
'$Actor2Code'
]
]
@ilantoren
ilantoren / build.gradle
Created February 8, 2022 10:23
Distance-Matrix with Google Cloud Scheduler
plugins {
id 'org.jetbrains.kotlin.jvm' version '1.6.10'
id 'java'
id 'application'
id("com.github.johnrengelman.shadow") version "6.0.0"
}
@ilantoren
ilantoren / PubSub.kt
Created February 8, 2022 08:39
Distance-Matrix with Google Cloud Scheuduler
package mongolovesdata
import com.google.cloud.functions.BackgroundFunction
import com.google.cloud.functions.Context
import com.google.type.DateTime
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.util.*
class PubSub() : BackgroundFunction<PubSub.PubSubMessage>{
@ilantoren
ilantoren / TravelApiResponse.kt
Created February 8, 2022 08:28
Distance-Matrix with Google Cloud Scheuduler
package mongolovesdata
import com.fasterxml.jackson.annotation.JsonIgnoreProperties
import com.fasterxml.jackson.annotation.JsonProperty
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
/*
Started with json to Kotlin conversion see sample.json in
the resource folder
*/
@JsonIgnoreProperties( ignoreUnknown = true)
@ilantoren
ilantoren / Process.kt
Last active February 8, 2022 07:33
Distance-Matrix with Google Cloud Scheuduler
package mongolovesdata
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import io.github.cdimascio.dotenv.Dotenv
import io.github.cdimascio.dotenv.dotenv
import io.ktor.client.*
import io.ktor.client.engine.cio.*
import io.ktor.client.features.json.*
import io.ktor.client.request.*
import io.ktor.client.statement.*
@ilantoren
ilantoren / create_usda.rs
Created January 25, 2022 09:36
Creating Usda objects from Document
async fn create_usda_set<'a>(client: &Client) {
let read_database = client.database(DB).collection::<Document>(SOURCE);
// as opposed to javascript/python/kotlin the projection is part of the find_options. The batch size is something you want to
// have with the async driver and arguably also with the sync since long queries can lose the cursor.
let find_options = FindOptions::builder().projection( doc!{"id":1, "product_name":1, "sources.url":1}).batch_size(1000).build();
// filter to look for records with a creator : usda and with a sources.url field
let filter = doc!{ "$and": [ { "creator": {"$regex": r"\busda\b", "$options": "i" }}, {"sources.url": {"$exists": 1}}]};
let cursor = read_database.find(filter, find_options).await;
match cursor {
Ok( mut cur) => {
@ilantoren
ilantoren / test_usda.rs
Last active January 25, 2022 08:21
Example of reading a struct from mongodb
#[derive(Serialize,Deserialize, Debug)]
pub struct Usda {
id: String,
product_name: String,
url: String
}
impl fmt::Display for Usda {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "id: {} product: {} url: {} ", self.id, self.product_name, self.url)
}
@ilantoren
ilantoren / RestaurantTable.tsx
Created December 12, 2021 06:06
realm query ream query
import React from "react";
import * as Realm from "realm-web";
import {app} from "./App";
import { Document } from 'mongodb'
import {DataGridRestaurants} from "./RestaurantDataGrid";
export const RestaurantTable: React.FC<{ neighborhood: String }> = ({neighborhood}) => {
let [data, setData] = React.useState<Document[] > ([])
let [last, setLast] = React.useState<String> ("")
@ilantoren
ilantoren / Restaurants.tsx
Last active December 12, 2021 13:56
Restaurant Dropdown
import TextField from '@mui/material/TextField';
import Autocomplete from '@mui/material/Autocomplete';
import {Container} from "@mui/material";
import * as Realm from 'realm-web';
import React from "react";
import {RestaurantTable} from "./RestaurantTable";
interface AutocompleteOption {
label: string;
}
@ilantoren
ilantoren / App.tsx
Last active December 12, 2021 13:48
Entry to Realm MERN React
import React from 'react';
import logo from '../logo.svg';
import './App.css';
import * as Realm from "realm-web";
import {ComboBox} from "./Restaurants";
import {Button} from "@mui/material";
const REALM_APP_ID = 'restaurantapp-bfnrh';
console.log("DATA", REALM_APP_ID )
export const app: Realm.App = new Realm.App({ id: REALM_APP_ID });