Skip to content

Instantly share code, notes, and snippets.

@s-zanella

s-zanella/Opacus bug report (>= 1.0)

Created October 4, 2022 11:01
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save s-zanella/b70308db3d6d1b1bf15a5a2c8a1cc525 to your computer and use it in GitHub Desktop.
Save s-zanella/b70308db3d6d1b1bf15a5a2c8a1cc525 to your computer and use it in GitHub Desktop.
Download ZIP
Opacus bug report: Length of `BatchSplittingSampler` with Poisson sampling
Raw
Opacus bug report (>= 1.0)
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"collapsed_sections": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"accelerator": "GPU",
"widgets": {
"application/vnd.jupyter.widget-state+json": {
"1b7264d186234500b74f840f7e5fa086": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HBoxModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HBoxModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HBoxView",
"box_style": "",
"children": [
"IPY_MODEL_3da4261dcdde4fab801c61e08aadd2ee",
"IPY_MODEL_68b4efba35654e13ba626a33f6e8f350",
"IPY_MODEL_a76e04bb7d654d25b4489888ab41312d"
],
"layout": "IPY_MODEL_94c8811791af403c999c2234ed575914"
}
},
"3da4261dcdde4fab801c61e08aadd2ee": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_f6ad0aa471af425fb15d686d6a746724",
"placeholder": "​",
"style": "IPY_MODEL_7d57435c4eb04f52af7daa73a023dbfb",
"value": "100%"
}
},
"68b4efba35654e13ba626a33f6e8f350": {
"model_module": "@jupyter-widgets/controls",
"model_name": "FloatProgressModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "FloatProgressModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "ProgressView",
"bar_style": "success",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_c519a423d9384b5ab8ceb6d00dd9f93b",
"max": 3124,
"min": 0,
"orientation": "horizontal",
"style": "IPY_MODEL_f8506a6294a340a7a4c4a228f054d5a8",
"value": 3124
}
},
"a76e04bb7d654d25b4489888ab41312d": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_90021be0ab754c8383d2208cf31abdbb",
"placeholder": "​",
"style": "IPY_MODEL_bba7c3d19e5a4c47aacba3f4c00d9679",
"value": " 3124/3124 [00:09<00:00, 344.49it/s]"
}
},
"94c8811791af403c999c2234ed575914": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"f6ad0aa471af425fb15d686d6a746724": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"7d57435c4eb04f52af7daa73a023dbfb": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"c519a423d9384b5ab8ceb6d00dd9f93b": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"f8506a6294a340a7a4c4a228f054d5a8": {
"model_module": "@jupyter-widgets/controls",
"model_name": "ProgressStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "ProgressStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"bar_color": null,
"description_width": ""
}
},
"90021be0ab754c8383d2208cf31abdbb": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"bba7c3d19e5a4c47aacba3f4c00d9679": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"9525fbd73a2240059bd81e4630f60638": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HBoxModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HBoxModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HBoxView",
"box_style": "",
"children": [
"IPY_MODEL_c7127e86f93d45c5a8ee385d9b4b0517",
"IPY_MODEL_51221454d11441f28e596f6d580a8255",
"IPY_MODEL_75de3546e2c840f29629a8a6795f091f"
],
"layout": "IPY_MODEL_24344d06f5bd4b538586b907598daa9f"
}
},
"c7127e86f93d45c5a8ee385d9b4b0517": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_fd4fbd95549a44f4b8522b586f6b30ea",
"placeholder": "​",
"style": "IPY_MODEL_a48285d3a26b496ba3ca14248f7f70dc",
"value": "100%"
}
},
"51221454d11441f28e596f6d580a8255": {
"model_module": "@jupyter-widgets/controls",
"model_name": "FloatProgressModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "FloatProgressModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "ProgressView",
"bar_style": "success",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_5787f126a763411f9ac7fd7adf5c5c37",
"max": 3125,
"min": 0,
"orientation": "horizontal",
"style": "IPY_MODEL_4825b35f525e44e396a176069c0ce66e",
"value": 3125
}
},
"75de3546e2c840f29629a8a6795f091f": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_39734c9a75e34a2b857dc0000709b2a3",
"placeholder": "​",
"style": "IPY_MODEL_adf696a5a49749bea392bffe7577997e",
"value": " 3125/3125 [00:01<00:00, 2564.01it/s]"
}
},
"24344d06f5bd4b538586b907598daa9f": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"fd4fbd95549a44f4b8522b586f6b30ea": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"a48285d3a26b496ba3ca14248f7f70dc": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"5787f126a763411f9ac7fd7adf5c5c37": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"4825b35f525e44e396a176069c0ce66e": {
"model_module": "@jupyter-widgets/controls",
"model_name": "ProgressStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "ProgressStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"bar_color": null,
"description_width": ""
}
},
"39734c9a75e34a2b857dc0000709b2a3": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"adf696a5a49749bea392bffe7577997e": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"b8858d7a1cac47b6941ef9eb4d68b0b3": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HBoxModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HBoxModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HBoxView",
"box_style": "",
"children": [
"IPY_MODEL_b5e7b298d0064f79acebc8d72efe7797",
"IPY_MODEL_fc28b377d66e481480d0b966961a5f26",
"IPY_MODEL_49206dfa3fdf4e33943bf32994ac7626"
],
"layout": "IPY_MODEL_aa6627c92a834daa8d3aebf39ec7e433"
}
},
"b5e7b298d0064f79acebc8d72efe7797": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_2b828d7bd7484070923ac20f7432e864",
"placeholder": "​",
"style": "IPY_MODEL_60b4a9c6279f4b09806dc59e1fbbd475",
"value": ""
}
},
"fc28b377d66e481480d0b966961a5f26": {
"model_module": "@jupyter-widgets/controls",
"model_name": "FloatProgressModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "FloatProgressModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "ProgressView",
"bar_style": "success",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_00fca4c4f8dc43d396beb7e2d40f061e",
"max": 6249,
"min": 0,
"orientation": "horizontal",
"style": "IPY_MODEL_b261fec9c4c64aae923cd58e1e05ff62",
"value": 6249
}
},
"49206dfa3fdf4e33943bf32994ac7626": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_1c9982b46eb44ed296e2454aa5817140",
"placeholder": "​",
"style": "IPY_MODEL_6dc7ef0c6bd44eea9ecdf6b293722ebe",
"value": " 7628/? [00:19<00:00, 424.53it/s]"
}
},
"aa6627c92a834daa8d3aebf39ec7e433": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"2b828d7bd7484070923ac20f7432e864": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"60b4a9c6279f4b09806dc59e1fbbd475": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"00fca4c4f8dc43d396beb7e2d40f061e": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"b261fec9c4c64aae923cd58e1e05ff62": {
"model_module": "@jupyter-widgets/controls",
"model_name": "ProgressStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "ProgressStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"bar_color": null,
"description_width": ""
}
},
"1c9982b46eb44ed296e2454aa5817140": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"6dc7ef0c6bd44eea9ecdf6b293722ebe": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"d4f526c345644f8b84b3e98610764fb6": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HBoxModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HBoxModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HBoxView",
"box_style": "",
"children": [
"IPY_MODEL_80fe9504dec249778039e524f674a708",
"IPY_MODEL_4730c06cbc474335aa4d57c87f656c7f",
"IPY_MODEL_806cd519c6cd4ac0bd11098f12284b8b"
],
"layout": "IPY_MODEL_9d42428444aa451bbda5c8f85ae43646"
}
},
"80fe9504dec249778039e524f674a708": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_18b12caf51af41bc99ad786621db54f0",
"placeholder": "​",
"style": "IPY_MODEL_133c2697d25040bc84c5c10dee102335",
"value": "100%"
}
},
"4730c06cbc474335aa4d57c87f656c7f": {
"model_module": "@jupyter-widgets/controls",
"model_name": "FloatProgressModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "FloatProgressModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "ProgressView",
"bar_style": "danger",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_1b1e8c58877b44779485494fc0955ee3",
"max": 7607,
"min": 0,
"orientation": "horizontal",
"style": "IPY_MODEL_aa26e68eb69c4360bedb297b0c91e611",
"value": 7575
}
},
"806cd519c6cd4ac0bd11098f12284b8b": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_3625a111cb8247e6896c5e1c607ada55",
"placeholder": "​",
"style": "IPY_MODEL_9b5f24d1dc2348eba591e723f72f0e30",
"value": " 7575/7607 [00:17<00:00, 440.11it/s]"
}
},
"9d42428444aa451bbda5c8f85ae43646": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"18b12caf51af41bc99ad786621db54f0": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"133c2697d25040bc84c5c10dee102335": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"1b1e8c58877b44779485494fc0955ee3": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"aa26e68eb69c4360bedb297b0c91e611": {
"model_module": "@jupyter-widgets/controls",
"model_name": "ProgressStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "ProgressStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"bar_color": null,
"description_width": ""
}
},
"3625a111cb8247e6896c5e1c607ada55": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"9b5f24d1dc2348eba591e723f72f0e30": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
}
}
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "6PI0sTeQXIqJ"
},
"source": [
"# Opacus bug repro notebook\n",
"\n",
"Reproduce a bug you experienced in this notebook. This way, we have a controlled environment and can run and reproduce things so that we can help you much faster.\n",
"\n",
"To help you go faster, let's prefill some things you will always want to have here such as dependencies."
]
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "OY0TWxFuXRwt",
"outputId": "fd1454ef-ac4b-4be1-c2e9-6b741adce6dc"
},
"source": [
"%%bash\n",
"apt install git-extras # Yup, Colab supports apt get! git-extras is helpful to eg fetch pull requests\n",
"\n",
"git clone https://github.com/pytorch/opacus.git # This will get you master\n",
"\n",
"# Interested in pulling a specific PR? It can be done like this:\n",
"# cd opacus\n",
"# git pr <PR_NUMBER> # For example, git pr 101 will fetch https://github.com/pytorch/opacus/pull/101\n",
"\n",
"cd opacus\n",
"pip install -e ."
],
"execution_count": 1,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Reading package lists...\n",
"Building dependency tree...\n",
"Reading state information...\n",
"git-extras is already the newest version (4.5.0-1).\n",
"The following package was automatically installed and is no longer required:\n",
" libnvidia-common-460\n",
"Use 'apt autoremove' to remove it.\n",
"0 upgraded, 0 newly installed, 0 to remove and 20 not upgraded.\n",
"Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
"Obtaining file:///content/opacus\n",
"Requirement already satisfied: functorch in /usr/local/lib/python3.7/dist-packages (from opacus==1.2.0) (0.2.1)\n",
"Requirement already satisfied: numpy>=1.15 in /usr/local/lib/python3.7/dist-packages (from opacus==1.2.0) (1.21.6)\n",
"Requirement already satisfied: torch>=1.8 in /usr/local/lib/python3.7/dist-packages (from opacus==1.2.0) (1.12.1+cu113)\n",
"Requirement already satisfied: scipy>=1.2 in /usr/local/lib/python3.7/dist-packages (from opacus==1.2.0) (1.7.3)\n",
"Requirement already satisfied: opt-einsum>=3.3.0 in /usr/local/lib/python3.7/dist-packages (from opacus==1.2.0) (3.3.0)\n",
"Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=1.8->opacus==1.2.0) (4.1.1)\n",
"Installing collected packages: opacus\n",
" Attempting uninstall: opacus\n",
" Found existing installation: opacus 1.2.0\n",
" Can't uninstall 'opacus'. No files were found to uninstall.\n",
" Running setup.py develop for opacus\n",
"Successfully installed opacus-1.2.0\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"\n",
"WARNING: apt does not have a stable CLI interface. Use with caution in scripts.\n",
"\n",
"fatal: destination path 'opacus' already exists and is not an empty directory.\n"
]
}
]
},
{
"cell_type": "code",
"metadata": {
"id": "RZOtdlUIYo3_"
},
"source": [
"exit() # Installing a new package requires restarting the environment"
],
"execution_count": 2,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "FaFmcQnsYz0f"
},
"source": [
"# Useful utils to speed you up\n",
"\n",
"To show us the behavior of your component, build the smallest possible end to end project.\n",
"\n",
"You can edit this as needed just so you don't start from scratch, but also feel free to ignore if it doesn't apply to your use case."
]
},
{
"cell_type": "code",
"source": [
"from statistics import mean\n",
"\n",
"def train(model, criterion, optimizer, train_loader, device=\"cuda:0\"):\n",
" accs = []\n",
" losses = []\n",
" for x, y in tqdm(train_loader):\n",
" x = x.to(device)\n",
" y = y.to(device)\n",
"\n",
" logits = model(x)\n",
" loss = criterion(logits, y)\n",
" loss.backward()\n",
"\n",
" optimizer.step()\n",
" optimizer.zero_grad()\n",
"\n",
" preds = logits.argmax(-1)\n",
" n_correct = float(preds.eq(y).sum())\n",
" batch_accuracy = n_correct / len(y)\n",
"\n",
" accs.append(batch_accuracy)\n",
" losses.append(float(loss))\n",
"\n",
" print(f\"Train Accuracy: {mean(accs):.6f}. Train Loss: {mean(losses):.6f}\")\n",
"\n",
"def test(model, test_loader, privacy_engine=None, device=\"cuda:0\"):\n",
" accs = []\n",
" with torch.no_grad():\n",
" for x, y in tqdm(test_loader):\n",
" x = x.to(device)\n",
" y = y.to(device)\n",
"\n",
" preds = model(x).argmax(-1)\n",
" n_correct = float(preds.eq(y).sum())\n",
" batch_accuracy = n_correct / len(y)\n",
"\n",
" accs.append(batch_accuracy)\n",
" print(f\"Test Accuracy: {mean(accs):.6f}\")"
],
"metadata": {
"id": "5RllsFY2Yts2"
},
"execution_count": 1,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "E64hS_O0Yvqn"
},
"source": [
"# Common imports\n",
"import os\n",
"import sys\n",
"from pathlib import Path\n",
"from typing import *\n",
"import numpy as np\n",
"import pandas as pd\n",
"\n",
"import torch\n",
"import torch.nn as nn\n",
"import torch.nn.functional as F\n",
"from torch.utils.data import Dataset, DataLoader\n",
"\n",
"from tqdm.autonotebook import tqdm\n",
"\n",
"import opacus"
],
"execution_count": 2,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "Z6DE9tNpZ9pL"
},
"source": [
"# Put here whatever args you have\n",
"\n",
"DEVICE = \"cuda:0\"\n",
"\n",
"BATCH_SZ = 16\n",
"FEATURE_DIM = 16\n",
"N_CLASSES = 3\n",
"\n",
"DATASET_LEN = 50_000"
],
"execution_count": 3,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "GVuM3nhpaEaT"
},
"source": [
"class DummyDataset(Dataset):\n",
" def __init__(self):\n",
" pass\n",
"\n",
" def __getitem__(self, i):\n",
" x = torch.randn([FEATURE_DIM]) # for categoricals like nn.Embedding you can do something like `torch.randint(0, VOC_SZ-1, (SEQ_LEN,), dtype=torch.long).to(device)`\n",
" y = 0\n",
" return x, y\n",
"\n",
" def __len__(self):\n",
" return DATASET_LEN"
],
"execution_count": 4,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "hJesOjOGaEyP"
},
"source": [
"train_ds = DummyDataset()\n",
"train_loader = DataLoader(train_ds, BATCH_SZ, shuffle=False)\n",
"\n",
"test_ds = DummyDataset()\n",
"test_loader = DataLoader(test_ds, BATCH_SZ, shuffle=False)\n",
"\n",
"x, y = next(iter(train_loader)) # if you want to get a sample batch to play with"
],
"execution_count": 5,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "ah2kk1xOaNC1",
"outputId": "834441df-1ca7-4f08-8e62-1da8e8464aaf"
},
"source": [
"model = nn.Linear(FEATURE_DIM, N_CLASSES).to(DEVICE)\n",
"criterion = nn.CrossEntropyLoss()\n",
"optimizer = torch.optim.SGD(model.parameters(), lr=0.1)\n",
"\n",
"privacy_engine = opacus.PrivacyEngine()\n",
"\n",
"model, optimizer, train_loader = privacy_engine.make_private(\n",
" module=model, \n",
" optimizer=optimizer,\n",
" data_loader=train_loader,\n",
" noise_multiplier=0.9, \n",
" max_grad_norm=1.0, \n",
")"
],
"execution_count": 6,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"/content/opacus/opacus/privacy_engine.py:142: UserWarning: Secure RNG turned off. This is perfectly fine for experimentation as it allows for much faster training performance, but remember to turn it on and retrain one last time before production with ``secure_mode`` turned on.\n",
" \"Secure RNG turned off. This is perfectly fine for experimentation as it allows \"\n"
]
}
]
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 121,
"referenced_widgets": [
"1b7264d186234500b74f840f7e5fa086",
"3da4261dcdde4fab801c61e08aadd2ee",
"68b4efba35654e13ba626a33f6e8f350",
"a76e04bb7d654d25b4489888ab41312d",
"94c8811791af403c999c2234ed575914",
"f6ad0aa471af425fb15d686d6a746724",
"7d57435c4eb04f52af7daa73a023dbfb",
"c519a423d9384b5ab8ceb6d00dd9f93b",
"f8506a6294a340a7a4c4a228f054d5a8",
"90021be0ab754c8383d2208cf31abdbb",
"bba7c3d19e5a4c47aacba3f4c00d9679"
]
},
"id": "fmYOuCfJa467",
"outputId": "56aa09eb-7cfe-46af-fcc4-73d4f87b2861"
},
"source": [
"train(model, criterion, optimizer, train_loader)"
],
"execution_count": 7,
"outputs": [
{
"output_type": "display_data",
"data": {
"text/plain": [
" 0%| | 0/3124 [00:00<?, ?it/s]"
],
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "1b7264d186234500b74f840f7e5fa086"
}
},
"metadata": {}
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py:1053: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
" warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n"
]
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"Train Accuracy: 0.997487. Train Loss: 0.022327\n"
]
}
]
},
{
"cell_type": "code",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 66,
"referenced_widgets": [
"9525fbd73a2240059bd81e4630f60638",
"c7127e86f93d45c5a8ee385d9b4b0517",
"51221454d11441f28e596f6d580a8255",
"75de3546e2c840f29629a8a6795f091f",
"24344d06f5bd4b538586b907598daa9f",
"fd4fbd95549a44f4b8522b586f6b30ea",
"a48285d3a26b496ba3ca14248f7f70dc",
"5787f126a763411f9ac7fd7adf5c5c37",
"4825b35f525e44e396a176069c0ce66e",
"39734c9a75e34a2b857dc0000709b2a3",
"adf696a5a49749bea392bffe7577997e"
]
},
"id": "pkggnjwra_uT",
"outputId": "c91b0700-900f-4dd8-b93e-5f303a18ae8c"
},
"source": [
"test(model, test_loader)"
],
"execution_count": 8,
"outputs": [
{
"output_type": "display_data",
"data": {
"text/plain": [
" 0%| | 0/3125 [00:00<?, ?it/s]"
],
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "9525fbd73a2240059bd81e4630f60638"
}
},
"metadata": {}
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"Test Accuracy: 1.000000\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"from opacus.utils.batch_memory_manager import BatchMemoryManager, wrap_data_loader\n",
"\n",
"def memory_safe_train(model, criterion, optimizer, train_loader, \n",
" max_physical_batch_size, total=None, device=\"cuda:0\"):\n",
" accs = []\n",
" losses = []\n",
"\n",
" with BatchMemoryManager(\n",
" data_loader=train_loader, \n",
" max_physical_batch_size=max_physical_batch_size, \n",
" optimizer=optimizer\n",
" ) as memory_safe_train_loader:\n",
" for x, y in tqdm(memory_safe_train_loader, total=total):\n",
" x = x.to(device)\n",
" y = y.to(device)\n",
"\n",
" optimizer.zero_grad()\n",
"\n",
" logits = model(x)\n",
" loss = criterion(logits, y)\n",
" loss.backward()\n",
" optimizer.step()\n",
"\n",
" preds = logits.argmax(-1)\n",
" n_correct = float(preds.eq(y).sum())\n",
" batch_accuracy = n_correct / len(y)\n",
"\n",
" accs.append(batch_accuracy)\n",
" losses.append(float(loss))\n",
"\n",
" print(f\"Train Accuracy: {mean(accs):.6f}. Train Loss: {mean(losses):.6f}\")"
],
"metadata": {
"id": "KJA-PUnzGlOI"
},
"execution_count": 10,
"outputs": []
},
{
"cell_type": "code",
"source": [
"model = nn.Linear(FEATURE_DIM, N_CLASSES).to(DEVICE)\n",
"optimizer = torch.optim.SGD(model.parameters(), lr=0.1)\n",
"\n",
"privacy_engine = opacus.PrivacyEngine()\n",
"\n",
"model, optimizer, train_loader = privacy_engine.make_private(\n",
" module=model, \n",
" optimizer=optimizer,\n",
" data_loader=train_loader,\n",
" noise_multiplier=0.9, \n",
" max_grad_norm=1.0, \n",
")"
],
"metadata": {
"id": "UeOaNwoRndBE"
},
"execution_count": 12,
"outputs": []
},
{
"cell_type": "code",
"source": [
"max_physical_batch_size = 8\n",
"\n",
"memory_safe_train_loader = wrap_data_loader(\n",
" data_loader=train_loader,\n",
" max_batch_size=max_physical_batch_size,\n",
" optimizer=optimizer,\n",
")\n",
"\n",
"print(f\"len(memory_safe_train_loader) = {len(memory_safe_train_loader)} largely underapproximates the expected length.\\n\"\n",
" \"The progress bar overflows\")\n",
"\n",
"memory_safe_train(model, criterion, optimizer, train_loader, \n",
" max_physical_batch_size=max_physical_batch_size)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 101,
"referenced_widgets": [
"b8858d7a1cac47b6941ef9eb4d68b0b3",
"b5e7b298d0064f79acebc8d72efe7797",
"fc28b377d66e481480d0b966961a5f26",
"49206dfa3fdf4e33943bf32994ac7626",
"aa6627c92a834daa8d3aebf39ec7e433",
"2b828d7bd7484070923ac20f7432e864",
"60b4a9c6279f4b09806dc59e1fbbd475",
"00fca4c4f8dc43d396beb7e2d40f061e",
"b261fec9c4c64aae923cd58e1e05ff62",
"1c9982b46eb44ed296e2454aa5817140",
"6dc7ef0c6bd44eea9ecdf6b293722ebe"
]
},
"id": "kX8ttlZRGnjT",
"outputId": "4d323728-125d-4728-ce78-3cc335458ffb"
},
"execution_count": 13,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"len(memory_safe_train_loader) = 6249 largely underapproximates the expected length.\n",
"The progress bar overflows\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
" 0%| | 0/6249 [00:00<?, ?it/s]"
],
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "b8858d7a1cac47b6941ef9eb4d68b0b3"
}
},
"metadata": {}
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"Train Accuracy: 0.994346. Train Loss: 0.026313\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"from scipy.stats import binom\n",
"\n",
"def approximate_length(memory_safe_train_loader):\n",
" self = memory_safe_train_loader.batch_sampler\n",
"\n",
" def F(k):\n",
" return binom(self.sampler.num_samples, self.sampler.sample_rate).cdf(k * self.max_batch_size) - \\\n",
" binom(self.sampler.num_samples, self.sampler.sample_rate).cdf((k - 1) * self.max_batch_size)\n",
"\n",
" expected_physical_batches = int(self.sampler.num_samples * self.sampler.sample_rate / self.max_batch_size)\n",
"\n",
" return int(\n",
" len(self.sampler) *\n",
" sum([i * F(i) for i in range(expected_physical_batches - 4, expected_physical_batches + 4)])\n",
" )"
],
"metadata": {
"id": "QYkedjFAG8Jx"
},
"execution_count": 14,
"outputs": []
},
{
"cell_type": "code",
"source": [
"model = nn.Linear(FEATURE_DIM, N_CLASSES).to(DEVICE)\n",
"optimizer = torch.optim.SGD(model.parameters(), lr=0.1)\n",
"\n",
"privacy_engine = opacus.PrivacyEngine()\n",
"\n",
"model, optimizer, train_loader = privacy_engine.make_private(\n",
" module=model, \n",
" optimizer=optimizer,\n",
" data_loader=train_loader,\n",
" noise_multiplier=0.9, \n",
" max_grad_norm=1.0, \n",
")"
],
"metadata": {
"id": "LtOevw6zngxp"
},
"execution_count": 15,
"outputs": []
},
{
"cell_type": "code",
"source": [
"memory_safe_train_loader = wrap_data_loader(\n",
" data_loader=train_loader,\n",
" max_batch_size=max_physical_batch_size,\n",
" optimizer=optimizer,\n",
")\n",
"\n",
"print(f\"approximate_length(memory_safe_train_loader) = {approximate_length(memory_safe_train_loader)} better approximates the expected length.\")\n",
"\n",
"memory_safe_train(model, criterion, optimizer, train_loader, \n",
" max_physical_batch_size=max_physical_batch_size, total=approximate_length(memory_safe_train_loader))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 84,
"referenced_widgets": [
"d4f526c345644f8b84b3e98610764fb6",
"80fe9504dec249778039e524f674a708",
"4730c06cbc474335aa4d57c87f656c7f",
"806cd519c6cd4ac0bd11098f12284b8b",
"9d42428444aa451bbda5c8f85ae43646",
"18b12caf51af41bc99ad786621db54f0",
"133c2697d25040bc84c5c10dee102335",
"1b1e8c58877b44779485494fc0955ee3",
"aa26e68eb69c4360bedb297b0c91e611",
"3625a111cb8247e6896c5e1c607ada55",
"9b5f24d1dc2348eba591e723f72f0e30"
]
},
"id": "DSE9KX87Hr3_",
"outputId": "b1ad56b4-5add-45ca-b88b-fff2f9f279e0"
},
"execution_count": 16,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"approximate_length(memory_safe_train_loader) = 7607 better approximates the expected length.\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
" 0%| | 0/7607 [00:00<?, ?it/s]"
],
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "d4f526c345644f8b84b3e98610764fb6"
}
},
"metadata": {}
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"Train Accuracy: 0.995118. Train Loss: 0.028032\n"
]
}
]
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment