rklaehn

package akkahttptest

import akka.actor.ActorSystem
import akka.http.Http
import akka.stream.FlowMaterializer
import akka.http.server._
import akka.http.marshalling.PredefinedToResponseMarshallers._
import akka.stream.scaladsl.{HeadSink, Source}

object Proxy extends App {

rklaehn

/// Run a get request that is complete (no dealing with resume) and hardcoded to the (name, hash) iroh collection
async fn run_get_request_collection(
    opts: iroh::dial::Options,
    hash: Hash,
) -> anyhow::Result<Stats> {
    let request = GetRequest::all(hash).into();
    let connection = iroh::dial::dial(opts).await?;
    let initial = fsm::start(connection, request);
    let connected = initial.next().await?;
    // on connected callback

rklaehn

Scenario

• Moderate size dataset (300GB)
• Too large to be stored entirely on end user hardware
• Seeder is not fast enough to serve all clients
• All users have small part of the dataset, but none have all
• User on consumer hadware want to browse with low latency

This scenario is mostly about content discovery, but it is a hard scenario that the hypercore team had issues with.

rklaehn

❯ ioreg -l|egrep 'gpu-core-count|AGXParameterBufferMax|GPUConfiguration|shared-region-base|class IOPlatformExpertDevice|perf-states|OS Build'
  |   "OS Build Version" = "21G115"
  +-o J316cAP  <class IOPlatformExpertDevice, id 0x10000026f, registered, matched, active, busy 0 (449072 ms), retain 38>
    | |   | |   "perf-states-sram" = <0000000016030000009e2c171603000080c5f71c1603000000b29f2616030000003c592e16030000008bef391903000000643f4d990300000000000016030000009e2c171603000080c5f71c1603000000b29f2616030000003c592e16030000008bef391603000000643f4d8d0300000000000016030000009e2c171603000080c5f71c1603000000b29f2616030000003c592e16030000008bef391c03000000643f4d990300000000000016030000009e2c171603000080c5f71c1603000000b29f2616030000003c592e16030000008bef391603000000643f4d8d030000>
    | |   | |   "perf-states-sram1" = <0000000016030000009e2c171603000080c5f71c1603000000b29f2616030000003c592e16030000008bef391603000000643f4d8d030000>
    | |   | |   "perf-states1" = <0000000090010000009e2c177702000080c5f71c900200000

rklaehn

//! Create a sudoku tester and make the test pass.

use std::{str::FromStr, fmt::Display};

struct Sudoku {
    grid: Vec<Vec<u8>>,
}

impl Sudoku {
    fn valid(&self) -> bool {

rklaehn

//! Decoding of dag-cbor blobs
//!
//! So far this is just used for figuring out links from a block for pinning
use cbor_event::{de::Deserializer, Len, Special, Type};
use cid::Cid;
use derive_more::{Display, From};
use std::{collections::BTreeSet, convert::TryFrom, error, io::Cursor};

const CBOR_TAG_LINK: u64 = 42;

rklaehn

/// This abomination only works if the underlying bytes are known in number (Sized),
/// can be moved around in memory (Unpin) and do no hold on to other things than the
/// underlying bytes ('static).
#[cfg(feature = "dataflow")]
impl<T: Abomonation + 'static + Sized + Unpin> Abomonation for ArcVal<T> {
    unsafe fn entomb<W: std::io::Write>(&self, write: &mut W) -> std::io::Result<()> {
        /* Since the value T has not yet been seen by abomonate (it is behind a pointer)
         * we need to fully encode it.
         */
        abomonation::encode(self.deref(), write)

rklaehn

Jul 24 13:25:22.534 DEBUG libp2p_gossipsub::behaviour: Subscribing to topic: discovery    
Jul 24 13:25:22.534 DEBUG libp2p_gossipsub::behaviour: Running JOIN for topic: TopicHash { hash: "discovery" }    
Jul 24 13:25:22.534 DEBUG libp2p_gossipsub::behaviour: RANDOM PEERS: Got 0 peers    
Jul 24 13:25:22.534 DEBUG libp2p_gossipsub::behaviour: JOIN: Inserting 0 random peers into the mesh    
Jul 24 13:25:22.535 DEBUG libp2p_gossipsub::behaviour: Completed JOIN for topic: TopicHash { hash: "discovery" }    
Jul 24 13:25:22.535  INFO libp2p_gossipsub::behaviour: Subscribed to topic: discovery    
Jul 24 13:25:22.537 DEBUG libp2p_gossipsub::behaviour: Subscribing to topic: /klk/prod/2020-01-09    
Jul 24 13:25:22.537 DEBUG libp2p_gossipsub::behaviour: Running JOIN for topic: TopicHash { hash: "/klk/prod/2020-01-09" }    
Jul 24 13:25:22.537 DEBUG libp2p_gossipsub::behaviour: RANDOM PEERS: Got 0 peers    
Jul 24 13:25:22.537 DEBUG libp2p_gossipsub::behaviour: JOIN: Inserting 0 random peers into the mesh    

rklaehn

use futures::prelude::*;
use std::{rc::Rc, task::{Context, Poll}, cell::RefCell, pin::Pin};

fn foo<'a>(c: &'a mut u64) -> impl Stream<Item = u64> + 'a {
    stream::unfold(10u64, move |max| {
        future::ready(if *c < max {
            *c += 1;
            Some((*c, max))
        } else {
            None

rklaehn

package akkahttptest

import akka.http.Http
import akka.stream.ActorFlowMaterializer
import akka.actor.ActorSystem
import akka.stream.scaladsl.{Sink, Source}
import akka.http.model._

object TestClient extends App {