Vindaar

More detailed thoughts about data extraction

This gist contains some ideas about using LLMs to extract data from papers (specifically related to biology, aging research and the like).

Just to quickly expand a bit on what I was trying to say when our meeting was cut off:

I think the LLM data extraction can be viewed as a problem tractable at 3 different layers:
1. purely text based, e.g. use `pdftotext` to turn a PDF into a text document, then use LLMs to summarize, extract, tag, ... papers in order to have machine readable data.

Vindaar

Table of Contents

1. How to compare
2. Compare witness data & zkey coefficients data SnarkJS vs CTT
3. buildABC results for A, B, C in SnarkJS vs CTT
4. A, B, C after FFT (NTT) transformations SnarkJS vs CTT
5. joinABC result SnarkJS vs CTT
6. Proof results g^A, g^B, g^C
7. Where is coeff, witness also used?

Vindaar

Creating a neural spike plot in ggplotnim

NOTE: once PR #59 is merged, the two examples below will be part of the recipes. Currently here: https://github.com/Vindaar/ggplotnim/blob/arraymancerBackend/recipes.org#creating-a-neural-spike-plot-in-ggplotnim

I’m not sure if this is what you had in mind, but I googled and found this:

https://pythontic.com/visualization/charts/spikerasterplot

Vindaar

Dplyr (R), Pandas (Python) & Nim data frame comparison

This comparison is inspired by the comparison here: https://gist.github.com/conormm/fd8b1980c28dd21cfaf6975c86c74d07

The Nim data frame implementation we use here is the Datamancer data frame.

Note that due to UFCS in Nim we can write the commands we present either similar to the Python notation as if the function were a method of the

Vindaar

import std / [strutils, strformat, tables, dynlib, os]
import noise, shell


import compiler/[llstream, renderer, types, magicsys, ast,
                 transf, # for code transformation (for -> while etc)
                 injectdestructors, # destructor injection
                 pathutils, # AbsoluteDir
                 modulegraphs] # getBody
import ./nimeval_dynlib_clean

Vindaar

import std / [strutils, strformat, tables, dynlib, os]
import noise, shell


import compiler/[llstream, renderer, types, magicsys, ast,
                 transf, # for code transformation (for -> while etc)
                 injectdestructors, # destructor injection
                 pathutils, # AbsoluteDir
                 modulegraphs] # getBody
import ./nimeval_dynlib

Vindaar

import noise, strutils, strformat, shell, tables, dynlib

proc printHelp() = echo ""

const procTmpl = """
{.push cdecl, exportc, dynlib.}
$#
{.pop.}
"""
const exprTmpl = """

Vindaar

import datamancer
import std / [math, complex]

const xn = 960
const yn = 960
const xmin = -2.0
const xmax = 0.6
const ymin = -1.5
const ymax = 1.5
const MAX_ITERS = 200

Vindaar

{"type":"Feature","geometry":{"type":"Point","coordinates":[18.9276,69.69,100]},"properties":{"meta":{"updated_at":"2022-12-22T14:42:02Z","units":{"air_pressure_at_sea_level":"hPa","air_temperature":"celsius","cloud_area_fraction":"%","precipitation_amount":"mm","relative_humidity":"%","wind_from_direction":"degrees","wind_speed":"m/s"}},"timeseries":[{"time":"2022-12-22T14:00:00Z","data":{"instant":{"details":{"air_pressure_at_sea_level":986.2,"air_temperature":-2.1,"cloud_area_fraction":86.7,"relative_humidity":81.6,"wind_from_direction":184.4,"wind_speed":6.1}},"next_12_hours":{"summary":{"symbol_code":"lightsnowshowers_day"}},"next_1_hours":{"summary":{"symbol_code":"lightsnow"},"details":{"precipitation_amount":0.2}},"next_6_hours":{"summary":{"symbol_code":"snow"},"details":{"precipitation_amount":2.2}}}},{"time":"2022-12-22T15:00:00Z","data":{"instant":{"details":{"air_pressure_at_sea_level":985.9,"air_temperature":-1.9,"cloud_area_fraction":84.9,"relative_humidity":80.7,"wind_from_direction":187.0,"wi

Vindaar

import ggplotnim, math
import arraymancer

const ε = 3
proc φ(r: float): float =
  result = exp(-pow((ε.float * r), 2.0))

proc toMatrix(n: int, start, stop: float): Tensor[float] =
  result = zeros[float]([n, n])
  let xs = linspace(start, stop, n)