2-parse-apify-data

2-parse-apify-data

import fs from 'fs';
import * as path from 'path';

import { AxiosResponse } from 'axios';
import dotenv from 'dotenv';
import { Configuration, CreateChatCompletionResponse, OpenAIApi } from 'openai';

// configure dotenv
dotenv.config();

function removeNewlineFromBeginning(str: string): string {
  // Check if the first character is a newline (\n)
  if (str.charAt(0) === '\n') {
    // Remove the first character and recursively call function to remove other newlines from beginning
    return removeNewlineFromBeginning(str.slice(1));
  }
  // If first character is not a newline, return the original string
  return str;
}

async function processFile(inputFilename: string) {
  const configuration = new Configuration({
    apiKey: process.env.OPENAI_API_KEY,
  });

  const openai = new OpenAIApi(configuration);

  if (!configuration.apiKey) {
    console.error('Check your Openai API key');
    return;
  }

  // Define the parsedData directory path
  const parsedDataDir = path.resolve('./tasks/data/parsedData');

  const rawdata = fs.readFileSync(path.join(parsedDataDir, inputFilename), 'utf8');

  let data: any[] = JSON.parse(rawdata);

  const parsedData = async (): Promise<any[]> => {
    return Promise.all(
      data.map(async (place) => {
        if (place.description?.length > 1) return place;

        const promptv2 = `Imagine you are a local expert writing for a travel website. Create a compelling and professional two-paragraph description for a local business called "${place.title}" located at "${place.street}, ${place.postalCode} Singapore." Highlight the unique qualities and services provided by the business. Ensure the paragraphs are concise yet informative, separated by a newline character.\n`;

        const systemMessageContent =
          'You are an AI assistant with extensive knowledge about local businesses. Your responses should be professional, concise, and provide useful information about each business listing. Provide a detailed, appealing description that highlights the unique services and ambiance of a business in two well-structured paragraphs. Avoid any subjective opinions or recommendations, and focus on factual information that would be helpful to visitors and potential customers. Your responses should be in perfect English, error-free, and formatted with paragraphs separated by a newline character.';

        try {
          // trigger OpenAI completion
          const response: AxiosResponse<CreateChatCompletionResponse> =
            await openai.createChatCompletion({
              model: 'gpt-4-1106-preview',
              messages: [
                {
                  role: 'system',
                  content: systemMessageContent,
                },
                {
                  role: 'user',
                  content: promptv2,
                },
              ],
              temperature: 0.7,
              top_p: 1,
              frequency_penalty: 0,
              presence_penalty: 0,
              max_tokens: 800,
              stream: false,
              n: 1,
            });

          // Check if response and necessary nested properties exist
          if (
            response &&
            response.data &&
            response.data.choices &&
            response.data.choices.length > 0 &&
            response.data.choices[0].message
          ) {
            const completionText = response.data.choices[0].message.content;
            // Ensure that completionText is a string and not undefined
            if (typeof completionText === 'string') {
              console.log([place.title, removeNewlineFromBeginning(completionText)]);
              place.description = removeNewlineFromBeginning(completionText);
            }
            return place;
          } else {
            // Handle the case where the expected data is not present
            console.error('The response from OpenAI did not contain the expected structure.');
            return place;
          }
        } catch (error: any) {
          if (error.response) {
            console.log(error.response.status);
            console.log(error.response.data);
          } else {
            console.log(error.message);
          }
        }
      }),
    );
  };

  const compiledData = await parsedData();
  console.log(`compiledData: ${compiledData.length}`);

  fs.writeFileSync(
    path.join(parsedDataDir, `final-${path.basename(inputFilename)}`),
    JSON.stringify(compiledData, null, 2),
    'utf-8',
  );
}

const command = process.argv[2];

if (!command) {
  console.error('Please provide a file name.');
  process.exit(1);
}

const filename = command;

processFile(filename);