Ensure keys and skus are set for syncing

commercetools.js

import { createClient } from '@commercetools/sdk-client';
import { createAuthMiddlewareForClientCredentialsFlow } from '@commercetools/sdk-middleware-auth';
import { createHttpMiddleware } from '@commercetools/sdk-middleware-http';
import { createQueueMiddleware } from '@commercetools/sdk-middleware-queue';
import { createRequestBuilder } from '@commercetools/api-request-builder';
import fetch from 'node-fetch';
import HttpsProxyAgent from 'https-proxy-agent';


export const fetchPatched = (url, options = {}) => {
  const instanceOptions = {
    ...options,
  };

  if (!options.agent && process.env.HTTP_PROXY) {
    instanceOptions.agent = new HttpsProxyAgent(process.env.HTTP_PROXY);
  }

  return fetch(url, instanceOptions);
};

export const Commercetools = ({
  clientId, clientSecret, projectKey, host, oauthHost, scopes, concurrency = 10,
}) => {
  const commercetools = {};

  commercetools.client = createClient({
    middlewares: [
      createAuthMiddlewareForClientCredentialsFlow({
        host: oauthHost,
        projectKey,
        credentials: {
          clientId,
          clientSecret,
        },
        fetch: fetchPatched,
        scopes,
      }),
      createQueueMiddleware({ concurrency }),
      createHttpMiddleware({ host, fetch: fetchPatched, enableRetry: true }),
    ],
  });

  commercetools.getRequestBuilder = () => createRequestBuilder({ projectKey });

  return commercetools;
};

commercetoolsErrors.js

export class ValidationError extends Error {
  constructor(message = 'Validation error', code = 400) {
    super(message);
    this.name = this.constructor.name;
    this.code = code;
  }
}
export class NotAuthorizedError extends Error {
  constructor(message = 'Not authorized', code = 401) {
    super(message);
    this.name = this.constructor.name;
    this.code = code;
  }
}
export class NotAuthenticatedError extends Error {
  constructor(message = 'Not authenticated', code = 403) {
    super(message);
    this.name = this.constructor.name;
    this.code = code;
  }
}
export class NotFoundError extends Error {
  constructor(message = 'not found', code = 404) {
    super(message);
    this.name = this.constructor.name;
    this.code = code;
  }
}

export class InternalServerError extends Error {
  constructor(message = 'Oops, something went wrong', code = 500) {
    super(message);
    this.name = this.constructor.name;
    this.code = code;
  }
}

export class UnMappedError extends Error {
  constructor(message = 'Some other error was returned!', code = 9000) {
    super(message);
    this.name = this.constructor.name;
    this.code = code;
  }
}

export const PROJECT_NOT_EXIST_ERROR = 'NO_PROJECT_OO1';

/**
 * Get a list of error messages from a ct error
 */
export const getErrorMessage = err => {
  let msg = JSON.stringify(err);
  if (err.body && err.body.errors && err.body.errors.length) {
    msg = err.body.errors[0].length ? err.body.errors[0].map(ctErr => ctErr.detailedErrorMessage).join(', ') : JSON.stringify(err.body.errors[0]);
  }
  return msg;
};

export const handleError = (rawMessage) => {
  const message = getErrorMessage(rawMessage);
  switch (rawMessage.statusCode) {
    case 400:
      throw new ValidationError(message);
    case 401:
      throw new NotAuthorizedError(message);
    case 403:
      throw new NotAuthenticatedError(message);
    case 404:
      throw new NotFoundError(message);
    case 500:
      throw new InternalServerError(message);
    default:
      console.error(rawMessage);
      throw new UnMappedError(message);
  }
};

ensureSkusForSync.js

/**
## Sync Caveats:


   * The tool expects that every variant has a sku and a key set.  I wrote a quick script to set those on all variants, but we probably want to include this into a lambda function.

   * The tool expects all State resources to have "initial:true" set on them, so it can set the state on those products/offers accordingly.

   * The tool does not handle offers/products that reference each other - It compiles a list of these as custom objects in the project.  This can be remedied by a script that takes that output, creates offers that contain all but the product reference attributes and then go back and update them, or run the sync again.

   * The tool ran into troubles with required fields that were not set.  (approvals, etc)  I had to set them as "pending" or remove the required constraint prior to syncing.

   * The tool is additive only, deletes are not synced.
**/

import '@babel/polyfill';
import { config } from 'dotenv';
import Promise from 'bluebird';
import uuid from 'uuid/v4';
import { Commercetools } from './commercetools';
import { ProductsService } from './products';

config();
const CONCURRENCY = 10;
const SLEEP_LENGTH = 750; // time in ms to wait after a request.

const commercetools = Commercetools({
  clientId: process.env.COMMERCE_TOOLS__CLIENT_ID,
  clientSecret: process.env.COMMERCE_TOOLS__CLIENT_SECRET,
  projectKey: process.env.COMMERCE_TOOLS__PROJECT_KEY,
  host: process.env.COMMERCE_TOOLS__API_HOST,
  oauthHost: process.env.COMMERCE_TOOLS__OAUTH_HOST,
  scopes: process.env.COMMERCE_TOOLS__SCOPES.split(' '),
});

const productsService = ProductsService({
  commercetools,
  sleepLength: SLEEP_LENGTH,
});
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));


const ensureSkuAndVariantKey = productProjection => {
  let variants = [productProjection.masterData.staged.variants];
  if (productProjection.masterData.staged.masterVariant) {
    variants = [productProjection.masterData.staged.masterVariant];
  }
  const actions = [];
  if (!variants[0].sku) {
    actions.push({
      action: 'setSku', variantId: 1, staged: false, sku: `${productProjection.key}-1`,
    });
  }
  if (!variants[0].key) {
    actions.push({
      action: 'setProductVariantKey', variantId: 1, staged: false, key: `${productProjection.key}-1`,
    });
  }
  if (actions.length) {
    return productsService.update(productProjection, actions);
  }
  return productProjection;
};


const ensureKey = productProjection => {
  if (productProjection.key) {
    return productProjection;
  } else {
    const key = uuid();
    return productsService.update(productProjection, [{ action: 'setKey', key }]);
  }
};
const ensureApprovals = productProjection => {
  let variants = [productProjection.masterData.staged.variants];
  if (productProjection.masterData.staged.masterVariant) {
    variants = [productProjection.masterData.staged.masterVariant];
  }
  const requiredFields = [
    'approvalStatus-OTTProductManagement',
    'approvalStatus-OTTAcqMkt',
    'approvalStatus-OTTRetentionMkt',
    'approvalStatus-OTTLegal',
    'approvalStatus-SuppliersPayments',
    'approvalStatus-Finance',
    'approvalStatus-DMG',
  ];
  const [masterVariant] = variants;
  const { attributes } = masterVariant;
  const actions = requiredFields.map(fieldname => {
    const match = attributes.find(x => x.name === fieldname);
    if (!match) {
      return {
        action: 'setAttributeInAllVariants',
        name: fieldname,
        value: 'pending',
      };
    }
    return false;
  }).filter(exists => exists);
  if (actions.length) {
    return productsService.update(productProjection, actions);
  }
  return productProjection;
};

const handlePage = results => {
  return Promise.map(results, async projection => {
    let curProjection = { ...projection };
    try {
      curProjection = await ensureKey(curProjection);
      curProjection = await ensureSkuAndVariantKey(curProjection);
      curProjection = await ensureApprovals(curProjection);
    } catch (e) {
      console.warn(e);
    }
  }, { concurrency: 40 });
};


const ensureAll = async () => {
  const options = {
    resourceTypeId: 'products',
    staged: false,
    sort: [{ by: 'id', direction: 'desc' }],
    perPage: 400,
  };
  let page = 1;

  let results = await productsService.fetch(options);
  while (results.length) {
    console.info(`Processing page ${page}`);
    // eslint-disable-next-line no-await-in-loop
    await handlePage(results);
    const lastId = results[results.length - 1].id;
    // eslint-disable-next-line no-await-in-loop
    results = await productsService.fetch({ ...options, where: [`id < "${lastId}"`] });
    page += 1;
  }
};


const main = async () => {
  console.info("I'm going to ensure keys and skus (and set null approvals) on all products.");
  await ensureAll();
};
main();

products.js

import { cloneDeep, isEqual } from 'lodash';
import { createSyncProducts } from '@commercetools/sync-actions';
import { handleError, NotFoundError } from './commercetoolsErrors';

const syncProducts = createSyncProducts([
  { type: 'categoryOrderHints', group: 'black' },
  { type: 'base', group: 'white' },
  { type: 'meta', group: 'white' },
  { type: 'references', group: 'white' },
  { type: 'prices', group: 'white' },
  { type: 'attributes', group: 'white' },
  { type: 'images', group: 'white' },
  { type: 'variants', group: 'white' },
]);

export const attributeBlacklist = [
  'qualifyingProductIds',
  'qualifyingProductTypes',
  'qualifyingProductFamily',
  'qualifyingProductCategories',
  'qualifyingProductStatuses',
  'excludedProductIds',
  'excludedProductTypes',
  'excludedProductFamily',
  'excludedProductCategories',
  'excludedProductStatuses',
  'includedProductIds',
  'includedProductTypes',
  'includedProductFamily',
  'includedProductCategories',
  'includedProductStatuses',
  'compatibleProductIds',
  'compatibleProductTypes',
  'compatibleProductFamily',
  'compatibleProductCategories',
  'compatibleProductStatuses',
  'eligibilityCustomerTypes',
  'eligibilityBusinessSegment',
  'eligibilityZips',
  'eligibilityCustomerSegments',
  'eligibilitySalesChannels',
  'eligibilityMinimumPurchaseAmounts',
];

const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));

const enumKVtoKeys = value => (value && value.key ? value.key : value);

export const toNameValueMap = nameValues => {
  return nameValues.reduce((o, item) => ({ ...o, [item.name]: item.value }), {});
};

export const enumValuesToKeys = values => {
  if (Array.isArray(values)) {
    const isNested = values.find(x => !Array.isArray(x) && typeof x.value !== 'undefined');
    if (isNested) {
      return values.map(x => ({ name: x.name, value: enumValuesToKeys(x.value) }));
    } else {
      return values.map(enumValuesToKeys);
    }
  }
  return enumKVtoKeys(values);
};


export const isNestedValueEqual = (plainValue, nestedRepresentation) => {
  try {
    const plainRep = enumValuesToKeys(nestedRepresentation);
    const equality = isEqual(plainRep, plainValue);
    return equality;
  } catch (e) {
    console.warn(e);
  }
  return false;
};


export const ProductsService = ({ commercetools, sleepLength }) => {
  const productsService = {};
  const { client, getRequestBuilder } = commercetools;

  /**
   * Retrieves a product by key
   */
  productsService.byKey = async ({ key }) => {
    const requestBuilder = getRequestBuilder();
    try {
      const result = await client.execute({
        uri: requestBuilder.products.parse({ key }).build(),
        method: 'GET',
      });
      await sleep(sleepLength);
      return result.body;
    } catch (err) {
      handleError(err);
    }
  };

  productsService.getByKeys = async ({ keys = [] }) => {
    if (!keys || !keys.length) {
      return [];
    }
    const requestBuilder = getRequestBuilder();
    const where = [`key in("${keys.join('", "')}")`];
    try {
      const result = await client.execute({
        uri: requestBuilder.products.parse({ where, perPage: 200 }).build(),
        method: 'GET',
      });
      await sleep(sleepLength);
      return result.body.results;
    } catch (err) {
      handleError(err);
    }
  };

  productsService.getOffersWithProducts = async ({ ids = [] }) => {
    if (!ids || !ids.length) {
      return [];
    }
    const requestBuilder = getRequestBuilder();
    const where = [`masterVariant(attributes(name="bundleProductIds" and value(id in("${ids.join('", "')}"))))`];
    try {
      const result = await client.execute({
        uri: requestBuilder.productProjections.parse({ where, perPage: 200, staged: true }).build(),
        method: 'GET',
      });
      await sleep(sleepLength);
      return result.body.results;
    } catch (err) {
      handleError(err);
    }
  };

  /**
   * Retrieves a product by id
   */
  productsService.byId = async ({ id }) => {
    const requestBuilder = getRequestBuilder();
    try {
      const result = await client.execute({
        uri: requestBuilder.products.parse({ id }).build(),
        method: 'GET',
      });
      await sleep(sleepLength);
      return result.body;
    } catch (err) {
      handleError(err);
    }
  };

  /**
   * Given a set of actions, updates a product
   */
  productsService.update = async (product, actions) => {
    const requestBuilder = getRequestBuilder();
    try {
      const result = await client.execute({
        uri: requestBuilder.products.parse({ id: product.id }).build(),
        method: 'POST',
        body: JSON.stringify({
          version: product.version,
          actions,
        }),
      });
      await sleep(sleepLength);
      return result.body;
    } catch (err) {
      console.error(JSON.stringify(err.body.errors), product.key, product.id);
      console.info('request actions', JSON.stringify(actions));
    }
  };

  /**
   * Fetches a certain page of results given a certain where clause
   */
  productsService.fetch = async ({
    where = [], page = 1, perPage = 200, resourceTypeId = 'products', sort = [{ by: 'createdAt', direction: 'desc' }],
  }) => {
    const requestBuilder = getRequestBuilder();
    try {
      const results = await client.execute({
        uri: requestBuilder[resourceTypeId].parse({
          where,
          page,
          perPage,
          sort,
        }).build(),
        method: 'GET',
      });
      await sleep(sleepLength);
      return results.body.results;
    } catch (err) {
      console.warn(`Error fetching ${resourceTypeId} where: ${where}`);
      console.info(err);
    }
  };

  productsService.fetchAll = async ({
    sort = [{ by: 'createdAt', direction: 'desc' }],
    where = [],
  }) => {
    let allResults = [];
    let page = 1;

    let results = await productsService.fetch({ where, page, sort });
    while (results.length) {
      allResults = [...allResults, ...results];
      page += 1;
      // eslint-disable-next-line no-await-in-loop
      results = await productsService.fetch({ where, page, sort });
    }
    return allResults;
  };
  productsService.delete = async (product) => {
    const requestBuilder = getRequestBuilder();
    try {
      const result = await client.execute({
        uri: `${requestBuilder.products.parse({ id: product.id }).build()}?version=${product.version}`,
        method: 'DELETE',
      });
      await sleep(sleepLength);
      return result.body;
    } catch (err) {
      console.error(JSON.stringify(err.body));
    }
  };

  /**
   * Given a ProductDraft, creates a product
   */
  productsService.create = async product => {
    const requestBuilder = getRequestBuilder();
    try {
      const result = await client.execute({
        uri: requestBuilder.products.build(),
        method: 'POST',
        body: JSON.stringify(product),
      });
      await sleep(sleepLength);
      return result.body;
    } catch (err) {
      console.warn(JSON.stringify(product));
      console.info(err.body.errors);
    }
  };

  /**
   * Creates a new product or updates an existing product.
   */
  productsService.createOrUpdate = async product => {
    // look up by key
    let existingProduct;
    if (product.key) {
      try {
        existingProduct = await productsService.byKey({ key: product.key });
      } catch (e) {
        // disregard any 404, as it can be expected.
        if (!(e instanceof NotFoundError)) {
          throw e;
        }
      }
    }
    if (existingProduct && existingProduct.id) {
      const draftToProductProjection = cloneDeep(existingProduct);
      draftToProductProjection.key = `${product.key}`;

      draftToProductProjection.masterData.staged = cloneDeep(product);
      // draftToProductProjection.masterData.staged.variants = [draftToProductProjection.masterData.staged.masterVariant]
      delete draftToProductProjection.masterData.staged.key;
      delete draftToProductProjection.masterData.staged.productType;
      delete draftToProductProjection.masterData.staged.publish;
      // hack to trick syncActions into thinking we've got the id already.
      draftToProductProjection.masterData.staged.masterVariant.id = 1;
      if (
        existingProduct.masterData
        && existingProduct.masterData.staged.masterVariant
      ) {
        draftToProductProjection.masterData.staged.masterVariant.id = existingProduct.masterData.staged.masterVariant.id;

        if (!draftToProductProjection.masterData.staged.masterVariant.prices) {
          draftToProductProjection.masterData.staged.masterVariant.prices = [];
        }

        // if there's existing prices with ids
        if (
          existingProduct.masterData.staged.masterVariant.prices
          && draftToProductProjection.masterData.staged.masterVariant.prices
        ) {
          // match prices by amount & dates

          const draftPrices = draftToProductProjection.masterData.staged.masterVariant.prices;

          if (
            existingProduct.masterData.staged.masterVariant.prices.length
            === draftPrices.length
          ) {
            draftPrices.forEach((price, index) => {
              price.id = existingProduct.masterData.staged.masterVariant.prices[
                index.id
              ];
            });
          } else {
            existingProduct.masterData.staged.masterVariant.prices.forEach(price => {
              // TODO: channel (to make this generic)
              // TODO: customergroup (to make this generic)
              const matchedPrice = draftPrices.find(x => x.validUntil === price.validUntil
                    && x.validFrom === price.validFrom
                    && x.value.centAmount === price.value.centAmount
                    && x.value.currencyCode === price.value.currencyCode);
              if (matchedPrice) {
                matchedPrice.id = price.id;
              }
            });
          }
        }
      }
      let actions = [];
      try {
        actions = syncProducts.buildActions(
          draftToProductProjection.masterData.staged,
          existingProduct.masterData.staged,
        );
      } catch (error) {
        console.error(
          `error during building sync actions for ${product.key}`,
          error,
        );
      }

      // for some reason it kept adding "setAttribute" for billingProductCode
      actions = actions.filter(x => {
        const matchingAttr = existingProduct.masterData.staged.masterVariant.attributes.find(attr => attr.name === x.name);
        if (
          x.action === 'setAttribute'
          && matchingAttr && (
            (x.value === matchingAttr.value)
            || (matchingAttr.value.key === x.value) // enums
            || isNestedValueEqual(x.value, matchingAttr.value)
          )
        ) {
          return false;
        } else if (x.action === 'setAttribute' && attributeBlacklist.indexOf(x.name) !== -1) {
          return false;
        } else if (x.action === 'transitionState' && product.state && x.state.id === product.state.id) {
          return false;
        }
        return true;
      }).map(x => {
        if (x.action === 'transitionState') {
          return { ...x, force: true };
        }
        return x;
      });

      existingProduct = await productsService.updateAssets(
        existingProduct,
        draftToProductProjection.masterData.staged.masterVariant.assets,
        existingProduct.masterData.staged.masterVariant.assets,
      );

      if (!actions.length) {
        return existingProduct;
      }

      console.info(`Updating product ${product.key}`, actions);
      return productsService.update(
        existingProduct,
        actions,
      );
    }
    console.info(`Creating product ${product.key}`);
    return productsService.create(product);
  };

  productsService.updateAssets = async (product, newAssets, existingAssets) => {
    const requestBuilder = getRequestBuilder();
    const assetActions = newAssets.map((newAsset) => {
      if (newAsset.custom && newAsset.custom.fields) {
        if (newAsset.custom.fields.longDesc === '') {
          delete newAsset.custom.fields.longDesc;
        }
        if (newAsset.custom.fields.shortDesc === '') {
          delete newAsset.custom.fields.shortDesc;
        }
      }
      return {
        action: 'addAsset',
        variantId: 1,
        asset: newAsset,
      };
    });
    try {
      if (assetActions.length === 0) {
        return product;
      }
      const deleteExistingAssetActions = existingAssets.map(existingAsset => {
        return {
          action: 'removeAsset',
          variantId: 1,
          assetId: existingAsset.id,
        };
      });
      const combinedAssetActions = [...deleteExistingAssetActions, ...assetActions];
      // console.log('Asset actions ', assetActions);
      const result = await client.execute({
        uri: requestBuilder.products.parse({ id: product.id }).build(),
        method: 'POST',
        body: JSON.stringify({
          version: product.version,
          actions: combinedAssetActions,
        }),
      });
      await sleep(sleepLength);
      return result.body;
    } catch (err) {
      console.error(JSON.stringify(err.body.errors));
      console.info('request assetactions', JSON.stringify(assetActions));
      return product;
    }
  };


  return productsService;
};
export default ProductsService;