Html Analysis

allVisibleElements.js

function allVisibleElements(root=document.body){
  const iter = document.createNodeIterator(root, NodeFilter.SHOW_ELEMENT, {
    acceptNode(e){
      if(e.offsetWidth && e.offsetHeight && e.getClientRects().length)
        return NodeFilter.FILTER_ACCEPT
    }
  })
  const nodes = []
  let node;
  while(node = iter.nextNode())
    nodes.push(node)
  return nodes
}
var allVisibleElementProps = allVisibleElements().map(elm => {
  var rect = elm.getBoundingClientRect(elm)
  var props = {
    type: elm.nodeName.toLowerCase(),
    top: rect.top,
    bottom: rect.bottom,
    left: rect.left,
    right: rect.right,
  }
  for(const attr of elm.attributes){
    switch(attr.name){
      case 'id':
        props.id = elm.id
        break
      case 'class':
        for(const name of elm.classList)
          props['.'+name] = true
        break
      case 'style':
        for(const style of elm.style)
          props['style.'+style] = elm.style[style]
        break
      default:
        props['@'+attr.name] = attr.value;
    }
  }
  return props
})
console.log(JSON.stringify(allVisibleElementProps))

notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cab944bf-41b4-46e8-908e-ddaedebc9c66",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6264ddb0-e718-4de2-9b93-d577e3eab949",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_json('./html.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "5df76c71-ec84-4c98-9d43-817df4bcf15d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "attr             val                                                                                                  \n",
       "type             div                                                                                                       True\n",
       "                 a                                                                                                         True\n",
       ".js-gps-track    1.0                                                                                                       True\n",
       "type             li                                                                                                        True\n",
       "left             707.0                                                                                                     True\n",
       "                                                                                                                          ...  \n",
       "@data-gps-track  site.switch({ item_type:11, target_site:553 }); posts_hot_network.click({ item_type:2, location:11 })    False\n",
       "bottom           1904.71875                                                                                               False\n",
       "@data-gps-track  site.switch({ item_type:11, target_site:475 }); posts_hot_network.click({ item_type:2, location:11 })    False\n",
       "bottom           1922.71875                                                                                               False\n",
       ".-ctas           1.0                                                                                                      False\n",
       "Length: 1819, dtype: bool"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t1 = df.unstack().dropna().reset_index()\n",
    "t1.columns = ['attr','elm','val']\n",
    "t1[['attr','val']].value_counts() > 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "701b3d92-b735-47f9-9bea-d2bbbfc42d06",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>attr</th>\n",
       "      <th>val</th>\n",
       "      <th>elm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>type</td>\n",
       "      <td>header</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>type</td>\n",
       "      <td>div</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>type</td>\n",
       "      <td>div</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>type</td>\n",
       "      <td>a</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>type</td>\n",
       "      <td>form</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3948</th>\n",
       "      <td>.js-footer-category-trigger</td>\n",
       "      <td>1.0</td>\n",
       "      <td>591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3949</th>\n",
       "      <td>.js-footer-category-trigger</td>\n",
       "      <td>1.0</td>\n",
       "      <td>593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3950</th>\n",
       "      <td>.js-footer-category-trigger</td>\n",
       "      <td>1.0</td>\n",
       "      <td>595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3951</th>\n",
       "      <td>.js-footer-category-trigger</td>\n",
       "      <td>1.0</td>\n",
       "      <td>597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3952</th>\n",
       "      <td>.js-footer-category-trigger</td>\n",
       "      <td>1.0</td>\n",
       "      <td>599</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3953 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             attr     val  elm\n",
       "0                            type  header    1\n",
       "1                            type     div    2\n",
       "2                            type     div    3\n",
       "3                            type       a    4\n",
       "4                            type    form    5\n",
       "...                           ...     ...  ...\n",
       "3948  .js-footer-category-trigger     1.0  591\n",
       "3949  .js-footer-category-trigger     1.0  593\n",
       "3950  .js-footer-category-trigger     1.0  595\n",
       "3951  .js-footer-category-trigger     1.0  597\n",
       "3952  .js-footer-category-trigger     1.0  599\n",
       "\n",
       "[3953 rows x 3 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Removing all attributes that only occur once\n",
    "t2 = t1.set_index(['attr','val']).loc[t1[['attr','val']].value_counts() > 1].reset_index()\n",
    "t2"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}