Skip to content

Instantly share code, notes, and snippets.

@sandipklevu

sandipklevu/klevu-sync-debug

Last active April 28, 2023 12:24
Show Gist options
  • Save sandipklevu/872979d8c3a2166245d71730f101ecd1 to your computer and use it in GitHub Desktop.
Save sandipklevu/872979d8c3a2166245d71730f101ecd1 to your computer and use it in GitHub Desktop.
Download ZIP
Gist for Klevu module 3.2.1 to trouble shoot debug issue
Raw
klevu-sync-debug
#!/usr/bin/env php
<?php
use Klevu\Search\Api\KlevuSyncRepositoryInterface;
use Klevu\Search\Api\MagentoProductSyncRepositoryInterface;
use Klevu\Search\Model\Klevu\Klevu;
use Magento\Eav\Model\Entity;
use Magento\Framework\App\Bootstrap;
use Magento\Store\Api\Data\StoreInterface;
if (PHP_SAPI !== 'cli') {
echo 'bin/klevu-sync-debug must be run as a CLI application';
exit(1);
}
$storeCode = isset($argv[1])
? trim((string)$argv[1])
: null;
if (!$storeCode) {
echo 'Please specify a store code';
exit(1);
}
try {
require __DIR__ . '/../app/bootstrap.php';
$bootstrap = Bootstrap::create(BP, $_SERVER);
$objectManager = $bootstrap->getObjectManager();
/** @var \Magento\Store\Model\StoreManagerInterface $storeManager */
$storeManager = $objectManager->get(\Magento\Store\Model\StoreManagerInterface::class);
} catch (\Exception $e) {
echo 'Autoload error: ' . $e->getMessage();
exit(1);
}
try {
$store = $storeManager->getStore($storeCode);
$storeId = (int)$store->getId();
} catch (\Exception $e) {
echo 'Error loading store: ' . $e->getMessage();
exit(1);
}
function diffMultiDimensionalArrays(array $array1, array $array2)
{
$jsonDiff = array_diff(
array_map('json_encode', $array1),
array_map('json_encode', $array2)
);
return array_map('json_decode', $jsonDiff);
}
/**
* @param $storeId
* @param $productIds
* @param $skipChildProducts
* @param $includeOosParents
* @param $magentoProductSyncRepository
*
* @return array
*/
function formatMagentoProductIds(
$storeId,
$productIds = [],
$skipChildProducts = false,
$includeOosParents = true,
$magentoProductSyncRepository
) {
$magentoProductIds = [];
if (!$productIds) {
return $magentoProductIds;
}
// getParentRelationsByChild
// $parentIds = $this->getParentRelationsByChild($productIds, $storeId, $includeOosParents);
$list = $magentoProductSyncRepository->getParentProductRelations($productIds, $storeId, $includeOosParents);
$parentIds = [];
foreach ($list as $row) {
if (!isset($parentIds[$row['product_id']])) {
$parentIds[$row['product_id']] = [];
}
$parentIds[$row['product_id']][] = $row;
}
unset($list);
// end getParentRelationsByChild
foreach ($productIds as $entityId) {
$entityParentIds = isset($parentIds[$entityId])
? $parentIds[$entityId]
: [];
foreach ($entityParentIds as $entityParentId) {
$magentoProductIds[] = [
'product_id' => $entityId,
'parent_id' => $entityParentId[Entity::DEFAULT_ENTITY_ID_FIELD],
];
}
if ($skipChildProducts) {
continue;
}
$magentoProductIds[] = [
'product_id' => $entityId,
'parent_id' => \Klevu\Search\Model\Product\MagentoProductActions::PARENT_ID_WHEN_NOT_VISIBLE,
];
}
unset($parentIds);
return $magentoProductIds;
}
/**
* As per 3.2.1 klevu module
*
* @param array $productIdsToDelete
* @param StoreInterface $store
* @param $includeOosProducts
* @param $magentoProductSyncRepository
*
* @return array
*/
function formatProductIdsToDelete(
array $productIdsToDelete,
StoreInterface $store,
$includeOosProducts = true,
$magentoProductSyncRepository
) {
$return = [];
/** @var MagentoProductSyncRepositoryInterface $magentoProductSyncRepository */
$parentProductIds = $magentoProductSyncRepository->getParentProductIds($store, $includeOosProducts);
foreach ($productIdsToDelete as $productIds) {
$productIds = (array)$productIds;
$parentId = $productIds[\Klevu\Search\Model\Klevu\Klevu::FIELD_PARENT_ID];
$productId = $productIds[\Klevu\Search\Model\Klevu\Klevu::FIELD_PRODUCT_ID];
$return[$parentId . '-' . $productId] = [
'parent_id' => $parentId,
'product_id' => $productId,
];
if (!(int)$parentId || ($parentId && in_array($parentId, $parentProductIds, true))
) {
continue;
}
$return[$productId] = [
'parent_id' => $parentId,
'product_id' => $productId,
];
}
return $return;
}
/** @var Magento\Framework\App\State $appState */
$appState = $objectManager->get(\Magento\Framework\App\State::class);
$appState->setAreaCode('frontend');
/** @var Magento\Store\Model\StoreManagerInterface $storeManager */
$storeManager = $objectManager->get(\Magento\Store\Model\StoreManagerInterface::class);
$storeManager->setCurrentStore($store);
/** @var KlevuSyncRepositoryInterface $klevuSyncRepository */
$klevuSyncRepository = $objectManager->get(KlevuSyncRepositoryInterface::class);
/** @var MagentoProductSyncRepositoryInterface $magentoProductSyncRepository */
$magentoProductSyncRepository = $objectManager->get(MagentoProductSyncRepositoryInterface::class);
/** @var \Klevu\Search\Model\Product\MagentoProductActionsInterface $magentoProductActions */
$magentoProductActions = $objectManager->get(\Klevu\Search\Model\Product\MagentoProductActionsInterface::class);
/** @var \Klevu\Search\Model\Klevu\ResourceModel\Klevu\Collection $klevuResourceCollection */
$klevuResourceCollection = $objectManager->get(\Klevu\Search\Model\Klevu\ResourceModel\Klevu\Collection::class);
/** @var \Klevu\Search\Helper\Config $klevuHelperConfig */
$klevuHelperConfig = $objectManager->get(\Klevu\Search\Helper\Config::class);
$includeOosProductsInSync = $klevuHelperConfig->includeOosProductsInSync($store);
/** @var \Klevu\Search\Model\Product\LoadAttribute $loadAttribute */
$loadAttribute = $objectManager->get(\Klevu\Search\Model\Product\LoadAttribute::class);
/**
* 3.2.1
* @param array $productIdsToAdd
* @param StoreInterface $store
* @param $includeOosProducts
* @param $magentoProductSyncRepository
*
* @return array
*/
function formatProductIdsToAdd(
array $productIdsToAdd,
StoreInterface $store,
$includeOosProducts = true,
$magentoProductSyncRepository
) {
$productsIds = [];
$parentProductIds = $magentoProductSyncRepository->getParentProductIds($store, $includeOosProducts);
foreach ($productIdsToAdd as $productIds) {
$productIds = (array)$productIds;
$parentId = $productIds[\Klevu\Search\Model\Klevu\Klevu::FIELD_PARENT_ID];
$productId = $productIds[\Klevu\Search\Model\Klevu\Klevu::FIELD_PRODUCT_ID];
if ((int)$parentId && !in_array($parentId, $parentProductIds, true)
) {
continue;
}
$productsIds[$parentId . '-' . $productId] = [
'parent_id' => $parentId,
'product_id' => $productId,
];
}
return $productsIds;
}
$productIds = [
'delete' => $magentoProductActions->deleteProductCollection($store),
'update' => $magentoProductActions->updateProductCollection($store),
'add' => $magentoProductActions->addProductCollection($store),
];
$maxSyncIds = [
'delete' => $klevuSyncRepository->getMaxSyncId($store),
'update' => $klevuSyncRepository->getMaxSyncId($store),
'add' => $magentoProductSyncRepository->getMaxProductId($store),
];
$colWidths = [20, 15, 15];
$divider = sprintf(
'+%s+%s+%s+',
str_repeat('-', $colWidths[0]),
str_repeat('-', $colWidths[1]),
str_repeat('-', $colWidths[2])
);
echo "ACTIONS SUMMARY" . PHP_EOL;
echo $divider . PHP_EOL;
echo sprintf(
'|%s|%s|%s|',
str_pad(' ACTION ', $colWidths[0], ' ', STR_PAD_RIGHT),
str_pad(' COUNT ', $colWidths[1], ' ', STR_PAD_LEFT),
str_pad(' MAX SYNC ID ', $colWidths[1], ' ', STR_PAD_LEFT)
) . PHP_EOL;
foreach ($maxSyncIds as $action => $maxSyncId) {
echo $divider . PHP_EOL;
echo sprintf(
'|%s|%s|%s|',
str_pad(' ' . $action . ' ', $colWidths[0], ' ', STR_PAD_RIGHT),
str_pad(' ' . number_format(count($productIds[$action])) . ' ', $colWidths[1], ' ', STR_PAD_LEFT),
str_pad(' ' . $maxSyncId . ' ', $colWidths[1], ' ', STR_PAD_LEFT)
) . PHP_EOL;
}
echo $divider . PHP_EOL;
echo PHP_EOL;
echo "PRODUCT IDS FOR ACTION" . PHP_EOL;
echo $divider . PHP_EOL;
echo sprintf(
'|%s|%s|%s|',
str_pad(' IDENTIFIER ', $colWidths[0], ' ', STR_PAD_RIGHT),
str_pad(' PARENT_ID ', $colWidths[1], ' ', STR_PAD_LEFT),
str_pad(' PRODUCT_ID ', $colWidths[1], ' ', STR_PAD_LEFT)
) . PHP_EOL;
foreach ($productIds as $rowTitle => $rowData) {
echo $divider . PHP_EOL;
echo sprintf('| %s |', str_pad($rowTitle, array_sum($colWidths))) . PHP_EOL;
foreach ($rowData as $identifier => $row) {
echo $divider . PHP_EOL;
echo sprintf(
'|%s|%s|%s|',
str_pad(' ' . $identifier . ' ', $colWidths[0], ' ', STR_PAD_RIGHT),
str_pad(' ' . $row['parent_id'] . ' ', $colWidths[1], ' ', STR_PAD_LEFT),
str_pad(' ' . $row['product_id'] . ' ', $colWidths[1], ' ', STR_PAD_LEFT)
) . PHP_EOL;
}
}
echo $divider . PHP_EOL;
echo PHP_EOL . PHP_EOL;
echo "PARENT PRODUCT IDS" . PHP_EOL;
$parentProductIds = $magentoProductSyncRepository->getParentProductIds($store);
echo implode(',', $parentProductIds) . PHP_EOL;
echo PHP_EOL . PHP_EOL;
// -- deleteProductCollection
echo "deleteProductCollection" . PHP_EOL;
echo "--------------------" . PHP_EOL . PHP_EOL;
$maxEntityId = $maxSyncIds['delete'];
if ($maxEntityId) {
$klevuSyncRepository->clearKlevuCollection();
$lastEntityId = 0;
$i = 0;
while ($lastEntityId < $maxEntityId) {
echo "Iteration: " . $i . PHP_EOL;
echo "- Max Entity Id: " . $maxEntityId . PHP_EOL;
echo "- Last Entity Id: " . $lastEntityId . PHP_EOL;
$klevuProductIds = $klevuSyncRepository->getProductIds($store, [], $lastEntityId);
echo "- Klevu Product Ids (" . count($klevuProductIds) . "): " . implode(',',
array_map('json_encode', $klevuProductIds)) . PHP_EOL;
if (!$klevuProductIds || ++$i >= \Klevu\Search\Model\Product\MagentoProductActions::MAX_ITERATIONS) {
echo PHP_EOL;
echo "* Exiting loop" . PHP_EOL;
break;
}
$lastEntityId = (int)max(array_column($klevuProductIds, \Klevu\Search\Model\Klevu\Klevu::FIELD_ENTITY_ID));
$productIdsToFilter = array_map(static function (array $product) {
return $product['product_id'];
}, $klevuProductIds);
array_walk($klevuProductIds, function (&$v) {
unset($v['row_id']);
});
echo "- Product Ids to Filter (" . count($productIdsToFilter) . "): " . implode(',', $productIdsToFilter)
. PHP_EOL;
$magentoProductIdsCollection = $magentoProductSyncRepository->getProductIdsCollection(
$store,
MagentoProductSyncRepositoryInterface::NOT_VISIBLE_INCLUDED
);
echo PHP_EOL . "- getProductIdsCollection Query: " . PHP_EOL .
$magentoProductIdsCollection->getSelect()->__toString()
. PHP_EOL;
echo "- Magento Product Ids Collection (" . $magentoProductIdsCollection->count() . "): " . implode(',',
$magentoProductIdsCollection->getColumnValues('entity_id')) . PHP_EOL;
$magentoProductIds = $magentoProductSyncRepository->getBatchDataForCollection(
$magentoProductIdsCollection,
$store,
$productIdsToFilter,
null
);
echo "- Magento Product Ids (" . count($magentoProductIds) . "): " . implode(',', $magentoProductIds) . PHP_EOL;
$magentoProductIds = formatMagentoProductIds(
$storeId,
$magentoProductIds,
false,
$includeOosProductsInSync,
$magentoProductSyncRepository
);
echo "- Magento Product Ids - after format (" . count($magentoProductIds) . "): "
. json_encode($magentoProductIds) . PHP_EOL;
$productIdsDiff = diffMultiDimensionalArrays($klevuProductIds, $magentoProductIds);
echo "- Product Ids Diff (" . count($productIdsDiff) . "): " . json_encode($productIdsDiff) . PHP_EOL;
$formattedProductIdsToDelete = formatProductIdsToDelete($productIdsDiff,
$store,
$includeOosProductsInSync,
$magentoProductSyncRepository);
echo "- Formatted Product Ids to Delete (" . count($formattedProductIdsToDelete) . "): "
. json_encode($formattedProductIdsToDelete) . PHP_EOL;
echo PHP_EOL . "-----" . PHP_EOL;
}
} else {
echo "No max entity for delete operation" . PHP_EOL;
}
echo PHP_EOL . PHP_EOL;
// -- updateProductCollection
echo "updateProductCollection" . PHP_EOL;
echo "--------------------" . PHP_EOL . PHP_EOL;
echo "SQL Query for filterProductsToUpdate: " . $klevuResourceCollection->filterProductsToUpdate()->getSelect()->__toString() . PHP_EOL;
$maxEntityId = $maxSyncIds['update'];
$klevuToUpdate = [];
if ($maxEntityId) {
$klevuSyncRepository->clearKlevuCollection();
$lastEntityId = 0;
$i = 0;
while ($lastEntityId < $maxEntityId) {
echo "Iteration: " . $i . PHP_EOL;
echo "- Max Entity ID: " . $maxEntityId . PHP_EOL;
echo "- Last Entity ID: " . $lastEntityId . PHP_EOL;
$klevuProductIds = $klevuSyncRepository->getProductIdsForUpdate(
$store,
$productIds['update'],
$lastEntityId
);
echo "- Klevu Product Ids (updateProductCollection) (" . count($klevuProductIds) . "): " . implode(',',
array_map('json_encode', $klevuProductIds)) . PHP_EOL;
if (!$klevuProductIds || ++$i >= \Klevu\Search\Model\Product\MagentoProductActions::MAX_ITERATIONS) {
echo PHP_EOL;
echo "* updateProductCollection Exiting loop" . PHP_EOL;
break;
}
$lastEntityId = (int)max(array_column(
$klevuProductIds,
\Klevu\Search\Model\Klevu\Klevu::FIELD_ENTITY_ID)
);
foreach ($klevuProductIds as $klevuItem) {
$parentFieldId = $klevuItem[\Klevu\Search\Model\Klevu\Klevu::FIELD_PARENT_ID];
$productFieldId = $klevuItem[\Klevu\Search\Model\Klevu\Klevu::FIELD_PRODUCT_ID];
$uniqueGroupKey = $parentFieldId . "-" . $productFieldId;
$klevuToUpdate[$uniqueGroupKey]["product_id"] = $productFieldId;
$klevuToUpdate[$uniqueGroupKey]["parent_id"] = $parentFieldId;
}
echo "- Klevu Product Ids - updateProductCollection (" . count($klevuToUpdate) . "): " . implode(',',
array_map('json_encode', $klevuToUpdate)) . PHP_EOL;
echo PHP_EOL . "-----" . PHP_EOL;
}
} else {
echo "No max entity for update operation" . PHP_EOL;
}
// -- addProductCollection
echo PHP_EOL . PHP_EOL;
echo "addProductCollection" . PHP_EOL;
echo "--------------------" . PHP_EOL . PHP_EOL;
$maxEntityId = $maxSyncIds['add'];
if ($maxEntityId) {
$klevuSyncRepository->clearKlevuCollection();
$batchedProductIds = [];
$productCollection = $magentoProductSyncRepository->getProductIdsCollection($store);
echo PHP_EOL . "- AddProductCollection (getProductIdsCollection): SQL Query: " . PHP_EOL
. $productCollection->getSelect()
->__toString()
. PHP_EOL;
$childProductCollection = $magentoProductSyncRepository->getChildProductIdsCollection($store);
echo PHP_EOL . "- AddProductCollection (getChildProductIdsCollection): SQL Query: " . PHP_EOL
. $childProductCollection->getSelect()->__toString()
. PHP_EOL;
$lastProductEntityId = 0;
$lastChildEntityId = 0;
$i = 0;
while ($lastProductEntityId < $maxEntityId || $lastChildEntityId < $maxEntityId) {
echo "Iteration: " . $i . PHP_EOL;
echo "- Max Entity ID : " . $maxEntityId . PHP_EOL;
echo "- Last Product Entity ID : " . $lastProductEntityId . PHP_EOL;
echo "- Last Child Entity ID : " . $lastChildEntityId . PHP_EOL;
$productIds = ($lastProductEntityId < $maxEntityId)
?
$magentoProductSyncRepository->getBatchDataForCollection($productCollection,
$store,
[],
$lastProductEntityId)
:
[];
echo "- Product Ids (" . count($productIds) . "): " . implode(',', $productIds) . PHP_EOL;
$childProductIds = ($lastChildEntityId < $maxEntityId)
?
$magentoProductSyncRepository->getBatchDataForCollection($childProductCollection,
$store,
[],
$lastChildEntityId)
:
[];
echo "- Child Product Ids (" . count($childProductIds) . "): " . implode(',', $childProductIds) . PHP_EOL;
if ((!$productIds && !$childProductIds)
|| ++$i >= \Klevu\Search\Model\Product\MagentoProductActions::MAX_ITERATIONS) {
echo PHP_EOL;
echo "* Exiting loop" . PHP_EOL;
break;
}
$lastProductEntityId = $productIds
? (int)max($productIds)
: $maxEntityId + 1;
$lastChildEntityId = $childProductIds
? (int)max($childProductIds)
: $maxEntityId + 1;
$magentoProductIdsToFilter = array_merge($productIds, $childProductIds);
echo "- Magento Product Ids To Filter (" . count($magentoProductIdsToFilter) . "): " . implode(',',
$magentoProductIdsToFilter) . PHP_EOL;
$magentoProductIds = array_merge(
formatMagentoProductIds($storeId,
$productIds,
false,
$includeOosProductsInSync,
$magentoProductSyncRepository
),
formatMagentoProductIds($storeId,
$childProductIds,
true,
$includeOosProductsInSync,
$magentoProductSyncRepository)
);
echo "- Magento Product Ids (" . count($magentoProductIds) . "): " . json_encode($magentoProductIds) . PHP_EOL;
$klevuProductIds = $klevuSyncRepository->getProductIds($store, $magentoProductIdsToFilter);
echo "- Klevu Product Ids (" . count($klevuProductIds) . "): " . json_encode($klevuProductIds) . PHP_EOL;
array_walk($klevuProductIds, function (&$v) {
unset($v['row_id']);
});
$productIdsDiff = diffMultiDimensionalArrays($magentoProductIds, $klevuProductIds);
echo "- Product Ids Diff (" . count($productIdsDiff) . "): " . json_encode($productIdsDiff) . PHP_EOL;
$formattedProductIdsToAdd = formatProductIdsToAdd($productIdsDiff,
$store,
$includeOosProductsInSync,
$magentoProductSyncRepository);
echo "- Formatted Product Ids to Add (" . count($formattedProductIdsToAdd) . "): "
. json_encode($formattedProductIdsToAdd) . PHP_EOL;
$loadAttributeCollection = $loadAttribute->loadProductDataCollection($formattedProductIdsToAdd, $storeId);
echo "- LoadAttributeCollection SQL Query : " . PHP_EOL .
$loadAttributeCollection->getSelect()->__toString()
. PHP_EOL;
echo PHP_EOL . "-----" . PHP_EOL;
}
} else {
echo "No max entity for add operation" . PHP_EOL;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment