dankolesnikov

import type { NextRequest } from "next/server";
import { isEmpty } from "lodash";
import { OPENAI_API_KEY } from "../../utils/variables";
import { OpenAICompletionsPayload, models } from "../../utils/types";

export const config = {
  runtime: "edge",
};

export default async function handler(req: NextRequest) {

dankolesnikov

@composite_solid
def get_data() -> constants.PandasDataFrame:
    local_path, s3_coordninates = io_solids.get_data_location()
    io_solids.read_csv_as_df_locally(local_path)
    io_solids.read_csv_as_df_s3(s3_coordninates)
    # dagster.core.errors.DagsterUserCodeProcessError: dagster.core.errors.DagsterInvalidDefinitionError: @composite_solid 'get_data' has unmapped output 'result'. Remove it or return a value from the appropriate solid invocation.

    
 
@composite_solid(output_defs=[

dankolesnikov

PipelineOptions options = PipelineOptionsFactory.create(); // configs
Pipeline p = Pipeline.create(options);

p.apply(TextIO.read().from("/shakespeare.txt")) // load the data
 .apply("ExtractWords", FlatMapElements
        .into(TypeDescriptors.strings())
        .via((String word) -> Arrays.asList(word.split("[^\\p{L}]+")))) // split lines where each element is a word - ParDo
 .apply(Count.<String>perElement()) // Convert PCollection into a Key/Value pair: word/# of occurences
 .apply("FormatResults", MapElements
    .into(TypeDescriptors.strings())

dankolesnikov

package edu.sjsu.cs185C;

import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
import java.util.NavigableMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;

dankolesnikov

package edu.sjsu.cs185C;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;

dankolesnikov

package edu.sjsu.cs185C;

import org.ojai.Document;
import org.ojai.DocumentStream;
import org.ojai.FieldPath;
import org.ojai.FieldSegment;
import org.ojai.store.*;

import java.util.Arrays;

dankolesnikov