judell

new method

func listDeployment(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) {
	conn, err := connect(ctx, d)
	if err != nil {
		plugin.Logger(ctx).Error("vercel_deployment.listDeployment", "connection_error", err)
		return nil, err
	}

judell

A query to cache (or not), should take ~1 (or ~10) secs.

with hn_ids as (
  select generate_series(1, 100) as id
),
hn_urls as (
  select
    'https://hacker-news.firebaseio.com/v0/item/' || h.id || '.json' as hn_api_url
 from

judell

I want to be able to leverage Steampipe's parallelism to fetch many items from Hacker News (or other plugins). I observed that although Hacker News implements both List and Get, only List is called in a JOIN context, so there's no opportunity to parallelize.

I made a plugin that (I think) works the same way as HN in this respect: https://github.com/judell/steampipe-plugin-hello. And I'm looking at different ways to make Steampipe employ the Get function.

This is the query result I am hoping for:

with ids as (
  select 1 as id union select 2 order by id
)

judell

fatal error: unexpected signal during runtime execution
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x7f4b08]

runtime stack:
runtime.throw({0x7f4d0285b2d8?, 0x0?})
	/opt/hostedtoolcache/go/1.19.0/x64/src/runtime/panic.go:1047 +0x5f fp=0x7ffddfb88268 sp=0x7ffddfb88238 pc=0x7f4d01db66bf
runtime.sigpanic()
	/opt/hostedtoolcache/go/1.19.0/x64/src/runtime/signal_unix.go:819 +0x369 fp=0x7ffddfb882b8 sp=0x7ffddfb88268 pc=0x7f4d01dccfc9

goroutine 17 [syscall, locked to thread]:

judell

--
-- PostgreSQL database dump
--

-- Dumped from database version 14.2
-- Dumped by pg_dump version 14.2 (Ubuntu 14.2-1.pgdg20.04+1+b1)

SET statement_timeout = 0;
SET lock_timeout = 0;
SET idle_in_transaction_session_timeout = 0;

judell

package hackernews

import (
	"context"

	"github.com/PaulRosset/go-hacknews"

	"github.com/turbot/steampipe-plugin-sdk/v3/plugin"
)

judell

+----------------+---------------------------------------------------------------+------------------------------------->
| twitter_user   | url                                                           | text                                >
+----------------+---------------------------------------------------------------+------------------------------------->
| idealoTech     | https://twitter.com/idealoTech/status/1524688985649516544     | Are you able to find creative soluti>
|                |                                                               |                                     >
|                |                                                               | Join our @codility Order #API Challe>
|                |                                                               |                                     >
|                |                                                               | #idealolife #codility #php          >
| idealoTech     | https://twitter.com/idealoTech/status

judell

-- find pyspark repos
with github_repos as (
  select 
    name, 
    owner_login, 
    stargazers_count 
  from 
    github_search_repository 
  where 
    query = 'pyspark' and name ~ 'pyspark'

judell

+----------------------+-------------------+------------------+
| name                 | owner_login       | stargazers_count |
+----------------------+-------------------+------------------+
| SynapseML            | microsoft         | 3297             |
| spark-nlp            | JohnSnowLabs      | 2725             |
| incubator-linkis     | apache            | 2524             |
| ibis                 | ibis-project      | 1805             |
| spark-py-notebooks   | jadianes          | 1455             |
| petastorm            | uber              | 1423             |
| awesome-spark        | awesome-spark     | 1314             |

judell

select 
  name, 
  owner_login, 
  stargazers_count 
from 
  github_search_repository 
where 
  query = 'pyspark' 
order by 
  stargazers_count desc