Solidity Visual Developer - Custom Unittest Stub

smart-contract-unittest-stub.md

About Testing

Testing helps to ensure that all code meets acceptance criteria and quality standards before the smart contract is deployed. Due to the immutability of smart contracts it is important to make sure a high code coverage is met, as well as testing all diferente edge scenarios. Nevertheless writing unit test can be time consuming, different developers might have different styles of testing. So I gave to myself the task to find a way to improve this process.

Solidity Visual Developer - VS Code extension

This extension contributes security centric syntax and semantic highlighting, a detailed class outline, specialized views, advanced Solidity code insights and augmentation to Visual Studio Code. It can be downloaded here

Review Features

• audit annotations/bookmarks - @audit - @audit-ok - (see below)
• generic interface for importing external scanner results - cdili json format (see below)
• codelens inline action: graph, report, dependencies, inheritance, parse, ftrace, flatten, generate unittest stub, function signature hashes, uml.

Custom Unittest Stub

One one hand, Solidity Visual Developer has many different features, but we are interested on generate unittest stub, it allows developers to generate a very basic unit test stub (truffle or hardhat) for a given smart contract. In the other hand, this extension does not allow to customize the unit test stub, but this can be hacked :p.

Intrusctions for macOS:

1.- Install the extension on Visual Studio Code https://marketplace.visualstudio.com/items?itemName=tintinweb.solidity-visual-auditor

2.- Locate the path where Visual Studio Code extensions are stored.

~/.vscode/extensions/tintinweb.solidity-visual-auditor-0.1.1/

3.- Modify the file /src/features/commands.js to fix an issue related to the parser instance.

#101
class Commands{ 
....
#114
  async generateUnittestStubForContract(document, contractName) {
      this._checkIsSolidity(document);
      let content;
      if(settings.extensionConfig().test.defaultUnittestTemplate==="hardhat"){
          // Fix wrong parser instance passed to the unit test stubs , it must be 'this.g_workspace'
          content = mod_templates.generateHardhatUnittestStubForContract(document, this.g_workspace, contractName);
      } else {
          // Fix wrong parser instance passed to the unit test stubs , it must be 'this.g_workspace'  
          content = mod_templates.generateUnittestStubForContract(document, this.g_workspace, contractName); 
      } 

      vscode.workspace.openTextDocument({content: content, language: "typescript"})
          .then(doc => vscode.window.showTextDocument(doc, vscode.ViewColumn.Beside));
  }  

4.- Modify the file /src/features/templates.js with the custom template. You can take as reference the template attached below. 5.- Re-start visual studio code to load the changes. 6.- Select the correct stub on Solidity Visual Develope extension configuration, select "hardhat".

Custom Template Features

• Generates a "setup" function to be called before all test cases. It incluse the deployment of the smart contract to test and any other dependency.
• List all events withing the smart contract so it is easier to remember which need to be tested.
• Generates a test case to validate public storage variabled.
• For each public or external function that mutates the storage, it generate two cases, correct and failure.
• For all public or external function that does not mutate the storage, it generate a test case.

Final Thoughts

Having an extension that helps to generate all this test cases really saves time and improves productivity, so you as a developer can focus on what is more important!, finding all those edge cases.

templates.js

"use strict";
/**
 * @author github.com/tintinweb
 * @license MIT
 *
 *
 * */

const vscode = require("vscode");
const filterNotVisibility = ["private", "internal"];
const filterStateMutability = ["view", "pure"];
const filterInitFn = ["constructor", "initialize", "init"];

function generateUnittestStubForContract(document, g_workspace, contractName) {
  let contract = {
    name: contractName,
    path: document.uri.fsPath,
  };

  if (!contractName) {
    //take first
    let sourceUnit = g_workspace.get(document.uri.fsPath);
    if (!sourceUnit || Object.keys(sourceUnit.contracts).length <= 0) {
      vscode.window.showErrorMessage(
        `[Solidity VA] unable to create unittest stub for current contract. missing analysis for source-unit: ${document.uri.fsPath}`
      );
      return;
    }

    contract.name = Object.keys(sourceUnit.contracts)[0];
  }

  let content = `
/**
 *
 * autogenerated by solidity-visual-auditor
 *
 * execute with:
 *  #> truffle test <path/to/this/test.js>
 *
 * */
var ${contract.name} = artifacts.require("${contract.path}");

contract('${contract.name}', (accounts) => {
    var creatorAddress = accounts[0];
    var firstOwnerAddress = accounts[1];
    var secondOwnerAddress = accounts[2];
    var externalAddress = accounts[3];
    var unprivilegedAddress = accounts[4]
    /* create named accounts for contract roles */

    before(async () => {
        /* before tests */
    })

    beforeEach(async () => {
        /* before each context */
    })

    it('should revert if ...', () => {
        return ${contract.name}.deployed()
            .then(instance => {
                return instance.publicOrExternalContractMethod(argument1, argument2, {from:externalAddress});
            })
            .then(result => {
                assert.fail();
            })
            .catch(error => {
                assert.notEqual(error.message, "assert.fail()", "Reason ...");
            });
        });

    context('testgroup - security tests - description...', () => {
        //deploy a new contract
        before(async () => {
            /* before tests */
            const new${contract.name} =  await ${contract.name}.new()
        })


        beforeEach(async () => {
            /* before each tests */
        })



        it('fails on initialize ...', async () => {
            return assertRevert(async () => {
                await new${contract.name}.initialize()
            })
        })

        it('checks if method returns true', async () => {
            assert.isTrue(await new${contract.name}.thisMethodShouldReturnTrue())
        })
    })
});
`;
  return content;
}
// utilities
function checkReservedIdentifiers(identifiers) {
  let decorations = [];
  if (!identifiers) {
    return decorations;
  }
  try {
    if (typeof identifiers.forEach !== "function") {
      identifiers = Object.values(identifiers);
    }
    identifiers.forEach(function (ident) {
      decorations.push(ident.name);
    });
  } catch (error) {
    decorations.push(error.message);
  }

  return decorations;
}
function censor(censor) {
  var i = 0;

  return function (key, value) {
    if (
      i !== 0 &&
      typeof censor === "object" &&
      typeof value == "object" &&
      censor == value
    )
      return "[Circular]";

    if (i >= 30)
      return "[Unknown]";

    ++i; 
    return value;
  };
}
function stringify(obj) {
  return JSON.stringify(obj, censor(obj), 2);
}
function tryFn(fn, ...args) {
  try {
    return fn(...args);
  } catch (error) {
    return error;
  }
}
//   content functions
function generateSetup(contract) {
  let initialize = ``;
  let constructor = ``;
  const fnInitialize = contract.functions.find((f) => f.name == "initialize");
  const fnConstructor = contract.functions.find((f) => f.name == "constructor");
  if (fnInitialize) {
    initialize = `// Initialize test contract.
    await ${contract.name}.initialize(${fnInitialize.parameters});`;
  }
  if (fnConstructor) {
    constructor = `${contract.instance} = await new ${contract.factory}(sa.default.signer).deploy(${fnConstructor.parameters})`;
  }
  const content = `
  const setup = async () => {
    const accounts = await ethers.getSigners()
    sa = await new StandardAccounts().initAccounts(accounts)
    mocks = await new ContractMocks().init(sa)
    nexus = mocks.nexus

    // Deploy dependencies of test contract.

    // Deploy test contract.
    ${constructor}
    ${initialize}

    // Add any approvals required for test contract.
    // await ${contract.instance}.approve("address", "uint256")
  };
`;
  return content;
}

function generateBehaviors(contract) {
  let content = "";
  try {
    let subcontractContent = [];
    for (let subcontract of contract.linearizedDependencies) {
      if (!subcontract || typeof subcontract !== "object") {
        continue;
      }
      if (subcontract.name == contract.name) {
        continue; //skip self
      }
      if (subcontract._node.kind === "interface") {
        continue; //skip inherited names from interfaces
      }
      let it = `it("should behave like ${subcontract.name} ", async () => {
                    await shouldBehaveLike${subcontract.name}(ctx)
                })`;
      subcontractContent.push(it);
    }
    content = `
    /**
        describe("behaviors", async () => {
            ${subcontractContent.map((c) => c).join(`\n                 `)}  
        })  
    */    
        `;
  } catch (error) {
    content = error.message;
  }
  return content;
}

function generateConstructor(contract) {
  const fnConstructor = contract.functions.find((f) => f.name == "constructor");
  let stateVarsContent = [];
  for (let stateVar of contract.stateVars) {
    stateVarsContent.push(
      `expect(await ${contract.instance}.${stateVar}(), "${stateVar}").to.eq("expected value");`
    );
  }
  let mappingsContent = [];
  // for (let mapping of contract.mappings) {
  //   mappingsContent.push(
  //     ` expect(await ${contract.instance}.${mapping}("key"), "${mapping}").to.eq("expected value");`
  //   );
  // }
  let content = `
        describe("constructor", async () => {
            it("should properly store valid arguments", async () => {
                ${stateVarsContent.map((c) => c).join(`\n                 `)}  
                ${mappingsContent.map((c) => c).join(`\n                  `)}  
            })
            it("fails if called with wrong arguments", async () => {
                // await expect(new ${contract.factory}(sa.default.signer).deploy(${fnConstructor.parameters}),"wrong arguments").to.be.revertedWith("expected error")
            })
        })      
        `;
  return content;
}
function generateInitialize(contract) {
  let content = ``;
  const fnInitialize = contract.functions.find((f) => f.name == "initialize");
  if (fnInitialize) {
    content = `
        describe("calling initialize", async () => {
            it("should default initializable values ", async () => {
                // expect(await ${contract.instance}.symbol(), "symbol").to.eq("expected value")
                // expect(await ${contract.instance}.name(), "name").to.eq("expected value")
            })
            it("fails if initialize is called more than once", async () => {
                await expect(${contract.instance}.initialize(${fnInitialize.parameters}),"init call twice").to.be.revertedWith("Initializable: contract is already initialized")
            })
        })        
        `;
  }
  return content;
}
function generateFnCase(contract, fn) {
  let caseContent = `
    // ${fn.str}
    describe("${fn.name}", async () => {
        beforeEach(async () => { /* before each context */})

        it('${fn.name} should ...', async () => {
            const tx = await ${contract.instance}.connect(sa.default.signer).${fn.name}(${fn.parameters})
            // Verify events, storage change, balance, etc.
            // await expect(tx).to.emit(${contract.instance}, "EVENT-NAME").withArgs("ARGUMENT 1", "ARGUMENT 2");

        });
        it('fails if ...', async () => {
            await expect(${contract.instance}.connect(sa.default.signer).${fn.name}(${fn.parameters}),"fails due to ").to.be.revertedWith("EXPECTED ERROR");
        });
    });
    `;
  return caseContent;
}
function generateStateMutabilityStorage(contract) {
  let fnContent = [];
  const storageFunctions = contract.functions.filter(
    (f) =>
      !filterStateMutability.includes(f.stateMutability) &&
      !filterInitFn.includes(f.name)
  );
  for (let fn of storageFunctions) {
    fnContent.push(generateFnCase(contract, fn, false));
  }
  let content = `${fnContent.map((c) => c).join("\n")}`;

  return content;
}
function generateStateMutabilityView(contract) {
  let fnContent = [];
  const readOnlyFunctions = contract.functions.filter(
    (f) =>
      filterStateMutability.includes(f.stateMutability) &&
      !filterInitFn.includes(f.name)
  );
  for (let fn of readOnlyFunctions) {
    let it = `
            it('${fn.name} should ...', async () => {
                const response = await ${contract.instance}.${fn.name}(${fn.parameters});
                expect(response, "${fn.name}").to.eq("expected value");    
            });
        `;
    fnContent.push(it);
  }
  let content = `
    describe("read only functions", async () => {
        beforeEach(async () => { /* before each context */})
        ${fnContent.map((c) => c).join("\n")}
    });
    `;
  return content;
}
function generateHardhatUnittestStubForContract(document,g_parser,contractName) {
  try {
    let contract = {
      name: contractName,
      path: document.uri.fsPath,
      instance:
        contractName.substring(0, 1).toLowerCase() + contractName.substring(1),
      factory: `${contractName}__factory`,
      functions: [],
    };

    if (!contractName) {
      //take first
      let sourceUnit = g_parser.sourceUnits[document.uri.fsPath];
      if (!sourceUnit || Object.keys(sourceUnit.contracts).length <= 0) {
        vscode.window.showErrorMessage(
          `[Solidity VA] unable to create hardhat-unittest stub for current contract. missing analysis for source-unit: ${document.uri.fsPath}`
        );
        return;
      }
      contract.name = Object.keys(sourceUnit.contracts)[0];
    }

    // add custom template
    let sourceUnit = g_parser.get(document.uri.fsPath);
    const contractObj = sourceUnit.contracts[contract.name];
    const functionNames = contractObj.functions.keys();
    contract.linearizedDependencies = contractObj.linearizedDependencies;
    contract.dependencies = contractObj.dependencies;
    contract.functions = [];
    for (let functionName of functionNames) {
      const fn = contractObj.functions[functionName];
      let node = fn._node;
      if (!filterNotVisibility.includes(node.visibility)) {
        const fnName = fn.name ? fn.name : "constructor";
        let str = "";
        const fnObj = {
          name: fnName,
          parameters: checkReservedIdentifiers(node.parameters),
          str,
          stateMutability: node.stateMutability,
        };
        contract.functions.push(fnObj);
      }
    }
    const events = [];
    for (var event of contractObj.events) {
      events.push(` // ${event.name}(${checkReservedIdentifiers(event.arguments)})`);
    }
    const stateVars = [];
    for (let _var in contractObj.stateVars) {
      if (!filterNotVisibility.includes(contractObj.stateVars[_var].visibility)) {
        stateVars.push(_var);
      }
    }
    contract.stateVars = stateVars;
    const mappings = [];
    for (let mapping in contractObj.mappings) {
      if (!filterNotVisibility.includes(contractObj.mappings[mapping].visibility)) {
        mappings.push(mapping);
      }
    }
    contract.mappings = mappings;

    let content = `
/**
 *
 * autogenerated by solidity-visual-auditor
 * template by @mStable
 *
 * execute with:
 *  #> yarn hardhat test <path/to/this/test.js>
 *
 * */
 
 import hre, { ethers } from "hardhat";
 import { expect } from "chai"
 import { Signer } from "ethers"
 import { simpleToExactAmount, BN } from "@utils/math"
 import { ${contract.name}, ${contract.factory}, MockNexus } from "types/generated"
 import { StandardAccounts, ContractMocks } from "@utils/machines"
 import { Account } from "types"

// -- Declare typescript interfaces for contracts --//
/**
 * interface SnapshotData {
 *  variableName: string;
 * }
 * */

 // Events to be tested
 ${events.map((e) => e).join("\n")}

describe('${contract.name}', () => {
  /* -- Declare shared variables -- */
  let sa: StandardAccounts
  let mocks: ContractMocks
  let nexus: MockNexus
  // Testing contract 
  let ${contract.instance}: ${contract.name}

  /* -- Declare shared functions -- */
    ${generateSetup(contract)}
    before("init contract",async () => {
        await setup()
    })
    beforeEach(async () => { /* before each context */})
    ${tryFn(generateBehaviors, contract)}
    ${tryFn(generateConstructor, contract)}
    ${tryFn(generateInitialize, contract)}
    ${tryFn(generateStateMutabilityStorage, contract)}
    ${tryFn(generateStateMutabilityView, contract)}    
});
`;
    return content;
  } catch (error) {
    return error.message;
  }
}
module.exports = {
  generateUnittestStubForContract: generateUnittestStubForContract,
  generateHardhatUnittestStubForContract:generateHardhatUnittestStubForContract,
};