Apply this expression to a path property to create a rectangle with inverted rounded corners (corners round inwards). At the beginning of the expression, you can set the corner radius and the two main points that form the rectangle (top left & bottom right). by default, those values are set to 'undefined' which will automatically create a rectan…

AfterEffectsRoundCornersRect.js

// USER SETTINGS

cornerRadius = undefined; // leave undefined to use comp size average;
topLeftCorner = undefined; // leave undefined to use comp zero point [0,0];
btmRightCorner = undefined; // leave undefined to use comp size point [compWidth,compHeight];

// EXPRESSION
function FindCirclePnt(cornerPnt, cornerRadius, angleFromCornerToCircle) {
 circlePntX = cornerRadius * Math.cos(degreesToRadians(angleFromCornerToCircle));
 circlePntY = cornerRadius * Math.sin(degreesToRadians(angleFromCornerToCircle));
 return [~~circlePntX, ~~circlePntY];
}

function Find2NewCornerPnts(iteration, cornerPnt, cornerRadius) {
 switch (iteration) {
   //tl

   case 0:
     newA = [cornerPnt[0], cornerPnt[1] + cornerRadius];
     newB = [cornerPnt[0] + cornerRadius, cornerPnt[1]];
     return [newA, newB];
     break;

   case 1:
     newA = [cornerPnt[0] - cornerRadius, cornerPnt[1]];
     newB = [cornerPnt[0], cornerPnt[1] + cornerRadius];
     return [newA, newB];
     break;

   case 2:
     newA = [cornerPnt[0], cornerPnt[1] - cornerRadius];
     newB = [cornerPnt[0] - cornerRadius, cornerPnt[1]];
     return [newA, newB];
     break;

   case 3:
     newA = [cornerPnt[0] + cornerRadius, cornerPnt[1]];
     newB = [cornerPnt[0], cornerPnt[1] - cornerRadius];
     return [newA, newB];
     break;

 }

}

function RoundARect(topLeftPnt, btmRightPnt, cornerRadius) {
 topLeftPnt = (topLeftPnt === undefined) ? [0, 0] : topLeftPnt;
 btmRightPnt = (btmRightPnt === undefined) ? [thisComp.width, thisComp.height] : btmRightPnt;
 cornerRadius = (cornerRadius === undefined) ? (0.02 * (thisComp.width + thisComp.height)) : cornerRadius;

 tl = topLeftPnt;
 tr = [btmRightPnt[0], topLeftPnt[1]];
 br = btmRightPnt;
 bl = [topLeftPnt[0], btmRightPnt[1]];

 AllCorners = [tl, tr, br, bl];

 allPntsArray = [];
 allITArray = [];
 allOTArray = [];

 for (i = 0; i < 4; i++) {
   angleFromCornerToCircle = 45 + (90 * i);
   angleOfCirlcePntOutTangent = angleFromCornerToCircle - 90;
   angleOfCirclePntInTangent = angleFromCornerToCircle + 90;
   currentCorner = AllCorners[i];
   newPntAB = Find2NewCornerPnts(i, currentCorner, cornerRadius);
   findCircle = FindCirclePnt(currentCorner, cornerRadius, angleFromCornerToCircle);
   newCirclePnt = currentCorner + findCircle
   newInTan = FindCirclePnt(newCirclePnt, (cornerRadius / 2), angleOfCirclePntInTangent);
   newOutTan = FindCirclePnt(newCirclePnt, (cornerRadius / 2), angleOfCirlcePntOutTangent);
		
   allPntsArray.push(newPntAB[0]);
   allPntsArray.push(newCirclePnt);
   allPntsArray.push(newPntAB[1]);
		
   allITArray.push([0, 0]);
   allITArray.push(newInTan);
   allITArray.push([0, 0]);
		
   allOTArray.push([0, 0]);
   allOTArray.push(newOutTan);
   allOTArray.push([0, 0]);
 }

 return createPath(points = allPntsArray, inTangents = allITArray, outTangents = allOTArray, isClosed = true)

}

RoundARect(topLeftCorner, btmRightCorner, cornerRadius);