Skip to content

Instantly share code, notes, and snippets.

@csjx
csjx / 01) spatial_indexing.py
Last active April 14, 2021 19:10
Exploring Parsl for indexing shapefiles
import os
import time
from typing import List
from pathlib import Path
import parsl
from parsl.app.app import python_app, bash_app
from parsl.executors import ThreadPoolExecutor
from parsl.executors import HighThroughputExecutor
from parsl.config import Config
from parsl.data_provider.files import File
@csjx
csjx / simplify-polygons.py
Created February 27, 2021 01:05
Use shapely.Polygon.simplify() to reduce ice wedge polygon vertices while maintaining geometry fidelity
import fiona
from pprint import pprint
from shapely.geometry import Point, Polygon
from descartes import PolygonPatch
import matplotlib.pyplot as plt
from pathlib import Path
data_folder = os.path.join(
Path.home(),
"Documents",

Set my token

export token="<token-goes-here>"

Create a 200MB file:

dd if=/dev/urandom of=200MB-file.txt bs=1M count=200
@csjx
csjx / update_system_metadata.py
Last active February 21, 2020 18:18
Example of listing all objects in a repository and updating the system metadata, in this case updating an AccessRule Subject string
#!/usr/bin/env python3
from d1_client.mnclient_2_0 import MemberNodeClient_2_0
import os
# Log in and get a token from https://search.dataone.org
# (or https://search-stage.test.dataone.org if using a test server)
# and set the token as a shell variable
token = os.getenv("token")
@csjx
csjx / LoginApp.java
Last active June 25, 2019 20:17
Using the JavaFX WebView embedded WebKit browser for ORCID authentication
package org.dataone.client.javafx;
import javafx.application.Application;
import javafx.scene.Scene;
import javafx.scene.layout.VBox;
import javafx.scene.web.WebView;
import javafx.stage.Stage;
import java.io.BufferedReader;
import java.io.IOException;
@csjx
csjx / create.py
Last active April 15, 2019 18:56
A quick example of a DataONE MNStorage.create() API call in python
#!/bin/env python3
from d1_client.mnclient_2_0 import MemberNodeClient_2_0
from d1_common.types import dataoneTypes
import uuid
import hashlib
import io
# Log in and get a token from https://knb.ecoinformatics.org
# or https://dev.nceas.ucsb.edu if using a a test server
@csjx
csjx / get_sysmeta.py
Last active March 19, 2019 21:41
Python example of DataONE CN.getSystemMetadata()
#!/bin/env python3
from d1_client.cnclient_2_0 import CoordinatingNodeClient_2_0
from d1_common.types import dataoneTypes
import os
# Set up the CN client with a token
token = os.environ["token"]
headers = {"Authorization": "Bearer " + token}
base_url = "https://cn.dataone.org/cn"
cn = CoordinatingNodeClient_2_0(base_url, headers = headers)
@csjx
csjx / required_apis.md
Last active April 24, 2018 22:12
DataONE Replication required API methods
Tier Version REST Function
1 1.0 GET /monitor/ping MNCore.ping()
1 1.0, 2.0 GET /log?[fromDate={fromDate}][&toDate={toDate}][&event={event}][&idFilter={idFilter}][&start={start}][&count={count}] MNCore.getLogRecords()
1 1.0 GET / and GET /node MNCore.getCapabilities()
1 1.0 GET /object/{id} MNRead.get()
1 1.0 GET /meta/{id} MNRead.getSystemMetadata()
1 1.0 HEAD /object/{id} MNRead.describe()
1 1.0 GET /checksum/{pid}[?checksumAlgorithm={checksumAlgorithm}] MNRead.getChecksum()
1 1.0 GET /object[?fromDate={fromDate}&toDate={toDate}&identifier={identifier}&formatId={formatId}&replicaStatus={replicaStatus}&start={start}&count={count}] MNRead.listObjects()
/**
*
*/
package org.dataone.tests;
import com.hazelcast.client.ClientConfig;
import com.hazelcast.client.HazelcastClient;
import com.hazelcast.config.GroupConfig;
import com.hazelcast.core.IQueue;
@csjx
csjx / filter_node_list.R
Created February 16, 2018 22:03
Filters the DataONE node list to extract certain node properties
library(xml2)
library(httr)
# Get the XML from the CN
nodes_xml <- GET("https://cn.dataone.org/cn/v2/node")
# Build an xml document object
node_document <- read_xml(nodes_xml)
# Filter out CNs and dedicated replica MNs