Skip to content

Instantly share code, notes, and snippets.

@Spriz
Created April 18, 2024 07:35
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save Spriz/c8fd8cbdd8a96530ca909838e320e24d to your computer and use it in GitHub Desktop.
Save Spriz/c8fd8cbdd8a96530ca909838e320e24d to your computer and use it in GitHub Desktop.
<?php
private function fillVendorProductsData(ConsoleIo $io)
{
$client = Typesense::getTypesenseClient(true);
/** @var \App\Model\Table\VendorProductsTable $table */
$table = $this->getTableLocator()->get('VendorProducts');
$collection = $client->getCollections()['vendor_products'];
assert($collection instanceof \Typesense\Collection);
$table->find()
->disableBufferedResults()
->disableHydration()
->disableAutoFields()
->select([
'id',
'vendor_id',
'product_category_number',
'product_number',
'product_number_without_first_four',
'barcode',
'name',
'price',
'unit_factor',
'unit_name',
'price_per_unit',
'expense_lines_count',
])
->chunk(150_000) //Uses ~408MB of memory
->each(function (array $batch, int $offset) use ($collection, $io) {
$io->info(sprintf('Processing %d vendor products (offset %d)', count($batch), $offset));
$rows = (new Collection($batch))
->map(function (array $product) {
return [
'id' => $product['id'],
'vendor_id' => $product['vendor_id'],
'product_category_number' => !empty($product['product_category_number']) ? $product['product_category_number'] : null,
'product_number' => !empty($product['product_number']) ? $product['product_number'] : null,
'product_number_without_first_four' => !empty($product['product_number_without_first_four']) ? $product['product_number_without_first_four'] : null,
'barcode' => !empty($product['barcode']) ? $product['barcode'] : null,
'name' => $product['name'],
'price' => $product['price'] >= 0.001 ? round($product['price'] * 1000, 0) : null,
'unit_factor' => $product['unit_factor'] >= 0.001 ? round($product['unit_factor'] * 1000, 0) : null,
'unit_name' => !empty($product['unit_name']) ? $product['unit_name'] : null,
'price_per_unit' => $product['price_per_unit'] >= 0.001 ? round($product['price_per_unit'] * 1000, 0) : null,
'expense_lines_count' => $product['expense_lines_count'],
];
})
->toArray();
$io->info('Saving batch of documents to Typesense: ' . count($rows));
$imports = $collection->getDocuments()->import($rows, ['action' => 'upsert']);
//If import has array element with success property set to false, throw exception
foreach ($imports as $import) {
if ($import['success'] !== true) {
throw new \RuntimeException('Import failed');
}
}
$io->info('Batch of documents saved to Typesense');
});
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment