Skip to content

Instantly share code, notes, and snippets.

@wrishel

wrishel/utilities_.idea_misc.xml

Created September 5, 2017 04:54
Show Gist options
  • Save wrishel/d77b49a12772cb0ebf4ce24fc5157be5 to your computer and use it in GitHub Desktop.
Save wrishel/d77b49a12772cb0ebf4ce24fc5157be5 to your computer and use it in GitHub Desktop.
Download ZIP
election transparency and TEVS utilities
Raw
utilities_.idea_misc.xml
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7.9 (/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7)" project-jdk-type="Python SDK" />
</project>
Raw
utilities_.idea_modules.xml
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/utilities.iml" filepath="$PROJECT_DIR$/.idea/utilities.iml" />
</modules>
</component>
</project>
Raw
utilities_.idea_utilities.iml
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
Raw
utilities_backup_pgm_.idea_backup_pgm.iml
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
<option name="projectConfiguration" value="py.test" />
<option name="PROJECT_TEST_RUNNER" value="py.test" />
</component>
</module>
Raw
utilities_backup_pgm_.idea_misc.xml
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7.9 (/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7)" project-jdk-type="Python SDK" />
</project>
Raw
utilities_backup_pgm_.idea_modules.xml
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/backup_pgm.iml" filepath="$PROJECT_DIR$/.idea/backup_pgm.iml" />
</modules>
</component>
</project>
Raw
utilities_backup_pgm_.idea_workspace.xml
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="91febe02-242d-4221-9eae-8b68228b218a" name="Default" comment="" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="TRACKING_ENABLED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="ExecutionTargetManager" SELECTED_TARGET="default_target" />
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file leaf-file-name="tevs_bkup.py" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/tevs_bkup.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="588">
<caret line="127" column="26" lean-forward="false" selection-start-line="127" selection-start-column="4" selection-end-line="127" selection-end-column="26" />
<folding>
<element signature="e#0#13#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Python Script" />
</list>
</option>
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/test_recharge/dircmper.py" />
<option value="$PROJECT_DIR$/bkup.py" />
<option value="$PROJECT_DIR$/dircmper.py" />
<option value="$PROJECT_DIR$/try_walk.py" />
<option value="$PROJECT_DIR$/tevs_bkup.py" />
</list>
</option>
</component>
<component name="ProjectFrameBounds">
<option name="x" value="365" />
<option name="y" value="288" />
<option name="width" value="1148" />
<option name="height" value="877" />
</component>
<component name="ProjectView">
<navigator currentView="ProjectPane" proportions="" version="1">
<flattenPackages />
<showMembers />
<showModules />
<showLibraryContents />
<hideEmptyPackages />
<abbreviatePackageNames />
<autoscrollToSource />
<autoscrollFromSource />
<sortByType />
<manualOrder />
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="ProjectPane">
<subPane>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="backup_pgm" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="backup_pgm" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="backup_pgm" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="backup_pgm" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="test_recharge" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
</subPane>
</pane>
<pane id="Scratches" />
<pane id="Scope" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="settings.editor.selected.configurable" value="project.propDebugger" />
</component>
<component name="RecentsManager">
<key name="MoveFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$" />
</key>
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="RunManager" selected="Python.bkup">
<configuration default="false" name="bkup" type="PythonConfigurationType" factoryName="Python" temporary="true">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="backup_pgm" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/tevs_bkup.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="true" />
<method />
</configuration>
<configuration default="true" type="CompoundRunConfigurationType" factoryName="Compound Run Configuration">
<method />
</configuration>
<configuration default="true" type="PythonConfigurationType" factoryName="Python">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="backup_pgm" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/bkup.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="true" />
<method />
</configuration>
<configuration default="true" type="Tox" factoryName="Tox">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="backup_pgm" />
<method />
</configuration>
<configuration default="true" type="tests" factoryName="Doctests">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="backup_pgm" />
<option name="SCRIPT_NAME" value="" />
<option name="CLASS_NAME" value="" />
<option name="METHOD_NAME" value="" />
<option name="FOLDER_NAME" value="" />
<option name="TEST_TYPE" value="TEST_SCRIPT" />
<option name="PATTERN" value="" />
<option name="USE_PATTERN" value="false" />
<method />
</configuration>
<configuration default="true" type="tests" factoryName="py.test">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="backup_pgm" />
<option name="_new_keywords" value="&quot;&quot;" />
<option name="_new_additionalArguments" value="&quot;&quot;" />
<option name="_new_target" value="&quot;.&quot;" />
<option name="_new_targetType" value="&quot;PATH&quot;" />
<method />
</configuration>
<list size="1">
<item index="0" class="java.lang.String" itemvalue="Python.bkup" />
</list>
<recent_temporary>
<list size="1">
<item index="0" class="java.lang.String" itemvalue="Python.bkup" />
</list>
</recent_temporary>
</component>
<component name="ShelveChangesManager" show_recycled="false">
<option name="remove_strategy" value="false" />
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="91febe02-242d-4221-9eae-8b68228b218a" name="Default" comment="" />
<created>1493684670031</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1493684670031</updated>
</task>
<servers />
</component>
<component name="ToolWindowManager">
<frame x="365" y="288" width="1148" height="877" extended-state="0" />
<editor active="true" />
<layout>
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.32640144" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32975295" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.2743259" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="false" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32942554" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.2496318" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32942554" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4806565" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
<window_info id="Data View" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
<window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" />
<window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32936507" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
</layout>
</component>
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
<component name="VcsManagerConfiguration">
<ignored-roots>
<path value="$PROJECT_DIR$/.." />
</ignored-roots>
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
<breakpoints-dialog>
<breakpoints-dialog />
</breakpoints-dialog>
<option name="time" value="26" />
</breakpoint-manager>
<watches-manager>
<configuration name="PythonConfigurationType">
<watch expression="src_stat.st_mtime" language="Python" />
<watch expression="tgt_stat.st_mtime" language="Python" />
<watch expression="src_stat.st_size" language="Python" />
<watch expression="tgt_stat.st_size" />
</configuration>
</watches-manager>
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/tevs_bkup.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding>
<element signature="e#0#13#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/tevs_bkup.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding>
<element signature="e#0#13#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/tevs_bkup.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding>
<element signature="e#0#13#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$APPLICATION_HOME_DIR$/helpers/pydev/pydevd.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="23760">
<caret line="1584" column="0" lean-forward="false" selection-start-line="1584" selection-start-column="0" selection-end-line="1584" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/tevs_bkup.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding>
<element signature="e#0#13#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/bkup.py" />
<entry file="file://$PROJECT_DIR$/dircmper.py" />
<entry file="file://$PROJECT_DIR$/tevs_bkup.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="135">
<caret line="12" column="22" lean-forward="false" selection-start-line="12" selection-start-column="22" selection-end-line="12" selection-end-column="22" />
<folding>
<element signature="e#0#13#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/bkup.py" />
<entry file="file://$PROJECT_DIR$/example.log">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="file:///Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/posixpath.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="167">
<caret line="72" column="0" lean-forward="false" selection-start-line="72" selection-start-column="0" selection-end-line="72" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="file:///Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/filecmp.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="575">
<caret line="139" column="30" lean-forward="true" selection-start-line="139" selection-start-column="30" selection-end-line="139" selection-end-column="30" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/dircmper.py" />
<entry file="file://$PROJECT_DIR$/bkup.py" />
<entry file="file://$APPLICATION_HOME_DIR$/helpers/pydev/pydevd.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="2861">
<caret line="1584" column="0" lean-forward="false" selection-start-line="1584" selection-start-column="0" selection-end-line="1584" selection-end-column="0" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/tevs_bkup.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="588">
<caret line="127" column="26" lean-forward="false" selection-start-line="127" selection-start-column="4" selection-end-line="127" selection-end-column="26" />
<folding>
<element signature="e#0#13#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</component>
</project>
Raw
utilities_backup_pgm_example.log
DEBUG:root:This message should go to the log file
INFO:root:So should this
WARNING:root:And this, too
DEBUG:root:This message should go to the log file
INFO:root:So should this
WARNING:root:And this, too
Raw
utilities_backup_pgm_test_recharge_input_exists_changed_date
This file is in the input and output, but has more content and a modify date in the input
This is the extra material that is in the input but not the output.
Raw
utilities_backup_pgm_test_recharge_input_exists_same_date_different_length
This file exists in the input and output, has the same modified data but different length.
And this is the extra line that creates a new length.
Raw
utilities_backup_pgm_test_recharge_input_folder_on_both_sides_exists_changed_date
This file is in the input and output, but has more content and a modify date in the input
This is the extra material that is in the input but not the output.
Raw
utilities_backup_pgm_test_recharge_input_folder_on_both_sides_exists_same_date_different_length
This file exists in the input and output, has the same modified data but different length.
Raw
utilities_backup_pgm_test_recharge_input_folder_on_both_sides_folder_on_both_sides_down1_exists_changed_date
This file is in the input and output, but has more content and a modify date in the input
This is the extra material that is in the input but not the output.
Raw
utilities_backup_pgm_test_recharge_input_folder_on_both_sides_folder_on_both_sides_down1_exists_same_date_different_length
This file exists in the input and output, has the same modified data but different length.
.
Raw
utilities_backup_pgm_test_recharge_input_newfile
This file is in the input but not the output
Raw
utilities_backup_pgm_test_recharge_output_exists_same_date_different_length
This file exists in the input and output, has the same modified data but different length.
Raw
utilities_backup_pgm_test_recharge_output_folder_on_both_sides_exists_changed_date
This file is in the input and output, but has more content and a modify date in the input
This is the extra material that is in the input but not the output.
Raw
utilities_backup_pgm_test_recharge_output_folder_on_both_sides_exists_same_date_different_length
This file exists in the input and output, has the same modified data but different length.
And this is the extra line that creates a new length.
Raw
utilities_backup_pgm_test_recharge_output_folder_on_both_sides_folder_on_both_sides_down1_exists_changed_date
This file is in the input and output, but has more content and a modify date in the input
This is the extra material that is in the input but not the output.
Raw
utilities_backup_pgm_test_recharge_output_folder_on_both_sides_folder_on_both_sides_down1_exists_same_date_different_length
This file exists in the input and output, has the same modified data but different length.
And this is the extra line that creates a new length.
Raw
utilities_backup_pgm_testdir_input_exists_changed_date
This file is in the input and output, but has more content and a modify date in the input
This is the extra material that is in the input but not the output.
Raw
utilities_backup_pgm_testdir_input_exists_same_date_different_length
This file exists in the input and output, has the same modified data but different length.
And this is the extra line that creates a new length.
Raw
utilities_backup_pgm_testdir_input_folder_on_both_sides_exists_changed_date
This file is in the input and output, but has more content and a modify date in the input
This is the extra material that is in the input but not the output.
Raw
utilities_backup_pgm_testdir_input_folder_on_both_sides_exists_same_date_different_length
This file exists in the input and output, has the same modified data but different length.
Raw
utilities_backup_pgm_testdir_input_folder_on_both_sides_folder_on_both_sides_down1_exists_changed_date
This file is in the input and output, but has more content and a modify date in the input
This is the extra material that is in the input but not the output.
Raw
utilities_backup_pgm_testdir_input_folder_on_both_sides_folder_on_both_sides_down1_exists_same_date_different_length
This file exists in the input and output, has the same modified data but different length.
.
Raw
utilities_backup_pgm_testdir_input_newfile
This file is in the input but not the output
Raw
utilities_backup_pgm_testdir_output_exists_same_date_different_length
This file exists in the input and output, has the same modified data but different length.
Raw
utilities_backup_pgm_testdir_output_folder_on_both_sides_exists_changed_date
This file is in the input and output, but has more content and a modify date in the input
This is the extra material that is in the input but not the output.
Raw
utilities_backup_pgm_testdir_output_folder_on_both_sides_exists_same_date_different_length
This file exists in the input and output, has the same modified data but different length.
And this is the extra line that creates a new length.
Raw
utilities_backup_pgm_testdir_output_folder_on_both_sides_folder_on_both_sides_down1_exists_changed_date
This file is in the input and output, but has more content and a modify date in the input
This is the extra material that is in the input but not the output.
Raw
utilities_backup_pgm_testdir_output_folder_on_both_sides_folder_on_both_sides_down1_exists_same_date_different_length
This file exists in the input and output, has the same modified data but different length.
And this is the extra line that creates a new length.
Raw
utilities_backup_pgm_tevs_bkup.py
import locale
import os
import re
import shutil
import sys
import time
"""
Copy all the files in the source path to the corresponding directory in the target path.
Recurses through subdirectories.
Skips files that are already in the target. However, if the file already in the target
has a different length or last modified date then overwrite it with the source file.
Exclude list (within) identifies file names that should not be copied, such as those beginning with '.'
"""
# recommas = re.compile('\d{1,3}')
def commanum(number):
"""Number is a number that can be expressed without exponential notation.
Output is a string that represents the number with separators according to
locale-based conventions. (E.g., for US commas separate thousands. For
France, spaces separate thousands. For India the grouping is not uniform.
We use this because the number of files copied can exceed 50K
"""
return locale.format('%d', number, True, True)
def excluded(s):
"""Test if s should be excluded from the target list"""
for rex in exclude_list:
if rex.match(s):
return True
return False
def make_set_files_only(p):
"""Return a set of partial paths to files identifed in a walk starting with
base path p.
The base path p is not included in the partial paths.
The set does not include files excluded by the exclude function."""
l = set()
for root, dirs, names in os.walk(p):
for name in names:
subpath = os.path.join(root, name).replace(p+os.sep, '', 1)
if (not excluded(name)) and (name not in dirs):
l.add(subpath)
sys.stdout.write('add: {:>12} {}\033[K\r' # clear cursor to end of line; \r not \n
.format(commanum(len(l)), os.path.join(root, name)))
print
return l
def ps(s):
"""Utility for sorted listing of the contents of a set."""
l = list(s)
l.sort()
for f in l: print f
def make_dirs(p):
"""Create any needed directories in the target."""
path, file = os.path.split(p)
if os.path.exists(path): return
os.makedirs(path)
return
# --------- main ---------
locale.setlocale(locale.LC_ALL, '') # for some reason this doesn't work in PyCharm
# locale.setlocale(locale.LC_ALL, 'fr_fr') # for testing
# debugging
src_path = '/Users/Wes/Dropbox/Programming/ElectionTransparency/backup_pgm/testdir/input'
tgt_path = '/Users/Wes/Dropbox/Programming/ElectionTransparency/backup_pgm/testdir/output'
# with test data at paths below this should copy 11 files
#
# src_path = '/home/tevs/2016-11'
# tgt_path = '/media/Seagate Backup Plus Drive/november 2016'
print 'Source: {}\nTarget: {}\n'.format(src_path, tgt_path)
exclude_list = [re.compile(ex) for ex in ['^\.']] # exclude hidden files (start with .)
tgt_set = make_set_files_only(tgt_path) # scan the files already at the target
src_set = make_set_files_only(src_path) # scan the files at the source
# Eliminate files already at the target unless the size or modification times are different.
#
n=0
rc = 0
for posbl_dupe in tgt_set & src_set:
src_stat = os.stat(os.path.join(src_path, posbl_dupe))
tgt_stat = os.stat(os.path.join(tgt_path, posbl_dupe))
n += 1
if src_stat.st_mtime == tgt_stat.st_mtime and src_stat.st_size == tgt_stat.st_size: # exclude files that match
rc += 1
sys.stdout.write('remove: {:>12} out of: {:>12} {}\033[K\r' # clear cursor to end of line; \r not \n
.format(commanum(n), commanum(rc), posbl_dupe))
src_set.remove(posbl_dupe)
print
# now copy the files with partial paths in src_set
#
l = list(src_set)
l.sort()
downcount = len(l) # the current test directories should show 11 files here
print "copy {} files".format(downcount)
for pp in l:
# show downcount and current partial path on the same line in console
# rather than scrolling
#
sys.stdout.write('copied: {:>12} {}\033[K\r' # clear cursor to end of line; \r not \n
.format(commanum(downcount), pp)) # clear to end of line
sys.stdout.flush()
downcount -= 1
src = os.path.join(src_path, pp)
dst = os.path.join(tgt_path, pp)
make_dirs(dst)
shutil.copy2(src, dst)
Raw
utilities_blankpages_.idea_workspace.xml
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="5a6c8898-9b4a-43fe-8eee-f2cd333b7fc1" name="Default" comment="">
<change type="DELETED" beforePath="$PROJECT_DIR$/../backup_pgm/bkup.by" afterPath="" />
<change type="DELETED" beforePath="$PROJECT_DIR$/.idea/misc.xml" afterPath="" />
<change type="DELETED" beforePath="$PROJECT_DIR$/.idea/vcs.xml" afterPath="" />
<change type="DELETED" beforePath="$PROJECT_DIR$/examinesmallpages.py" afterPath="" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/../b/b.sql" afterPath="$PROJECT_DIR$/../b/b.sql" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/../b/import_ballots_n_voteops.sql" afterPath="$PROJECT_DIR$/../b/import_ballots_n_voteops.sql" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/../psql_out.txt" afterPath="$PROJECT_DIR$/../psql_out.txt" />
</list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="TRACKING_ENABLED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="FileEditorManager">
<leaf />
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$/.." />
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/examinesmallpages.py" />
</list>
</option>
</component>
<component name="ProjectFrameBounds">
<option name="x" value="240" />
<option name="y" value="113" />
<option name="width" value="1440" />
<option name="height" value="874" />
</component>
<component name="ProjectLevelVcsManager" settingsEditedManually="true" />
<component name="ProjectView">
<navigator currentView="ProjectPane" proportions="" version="1">
<flattenPackages />
<showMembers />
<showModules />
<showLibraryContents />
<hideEmptyPackages />
<abbreviatePackageNames />
<autoscrollToSource />
<autoscrollFromSource />
<sortByType />
<manualOrder />
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="ProjectPane">
<subPane>
<expand>
<path>
<item name="blankpages" type="b2602c69:ProjectViewProjectNode" />
<item name="blankpages" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
<pane id="Scope" />
<pane id="Scratches" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="settings.editor.selected.configurable" value="fileTemplates" />
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="ShelveChangesManager" show_recycled="false">
<option name="remove_strategy" value="false" />
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="5a6c8898-9b4a-43fe-8eee-f2cd333b7fc1" name="Default" comment="" />
<created>1503963724442</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1503963724442</updated>
</task>
<task id="LOCAL-00001" summary="Initial commit of separate blankpages project">
<created>1503965867792</created>
<option name="number" value="00001" />
<option name="presentableId" value="LOCAL-00001" />
<option name="project" value="LOCAL" />
<updated>1503965867792</updated>
</task>
<option name="localTasksCounter" value="2" />
<servers />
</component>
<component name="ToolWindowManager">
<frame x="240" y="113" width="1440" height="874" extended-state="0" />
<layout>
<window_info id="Project" active="true" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.24964234" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.32936507" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" />
<window_info id="Data View" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" />
<window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
</layout>
</component>
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
<component name="VcsManagerConfiguration">
<MESSAGE value="Initial commit of separate blankpages project" />
<option name="LAST_COMMIT_MESSAGE" value="Initial commit of separate blankpages project" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager />
<watches-manager />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/examinesmallpages.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="236">
<caret line="25" column="13" lean-forward="true" selection-start-line="25" selection-start-column="13" selection-end-line="25" selection-end-column="13" />
<folding />
</state>
</provider>
</entry>
</component>
</project>
Raw
utilities_cropper_.idea_cropper.iml
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="Unittests" />
</component>
</module>
Raw
utilities_cropper_.idea_misc.xml
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7.13 (/usr/local/Cellar/python/2.7.13/Frameworks/Python.framework/Versions/2.7/bin/python2.7)" project-jdk-type="Python SDK" />
</project>
Raw
utilities_cropper_.idea_modules.xml
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/cropper.iml" filepath="$PROJECT_DIR$/.idea/cropper.iml" />
</modules>
</component>
</project>
Raw
utilities_cropper_.idea_workspace.xml
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="97259491-12e8-4b21-8042-1c50a8b8bff0" name="Default" comment="" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="TRACKING_ENABLED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="ExecutionTargetManager" SELECTED_TARGET="default_target" />
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file leaf-file-name="cropper.py" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/cropper.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="300">
<caret line="20" column="0" lean-forward="false" selection-start-line="20" selection-start-column="0" selection-end-line="20" selection-end-column="0" />
<folding>
<element signature="e#0#21#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Python Script" />
</list>
</option>
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/cropper.py" />
</list>
</option>
</component>
<component name="ProjectFrameBounds">
<option name="x" value="22" />
<option name="y" value="223" />
<option name="width" value="1806" />
<option name="height" value="971" />
</component>
<component name="ProjectView">
<navigator currentView="ProjectPane" proportions="" version="1">
<flattenPackages />
<showMembers />
<showModules />
<showLibraryContents />
<hideEmptyPackages />
<abbreviatePackageNames />
<autoscrollToSource />
<autoscrollFromSource />
<sortByType />
<manualOrder />
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="ProjectPane">
<subPane>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="cropper" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="cropper" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
</subPane>
</pane>
<pane id="Scope" />
<pane id="Scratches" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="settings.editor.selected.configurable" value="configurable.group.appearance" />
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="RunManager" selected="Python.Unnamed">
<configuration default="true" type="PythonConfigurationType" factoryName="Python">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="cropper" />
<option name="SCRIPT_NAME" value="" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<method />
</configuration>
<configuration default="true" type="Tox" factoryName="Tox">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="cropper" />
<method />
</configuration>
<configuration default="true" type="tests" factoryName="Doctests">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="cropper" />
<option name="SCRIPT_NAME" value="" />
<option name="CLASS_NAME" value="" />
<option name="METHOD_NAME" value="" />
<option name="FOLDER_NAME" value="" />
<option name="TEST_TYPE" value="TEST_SCRIPT" />
<option name="PATTERN" value="" />
<option name="USE_PATTERN" value="false" />
<method />
</configuration>
<configuration default="true" type="tests" factoryName="Unittests">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="cropper" />
<option name="_new_additionalArguments" value="&quot;&quot;" />
<option name="_new_target" value="&quot;.&quot;" />
<option name="_new_targetType" value="&quot;PATH&quot;" />
<method />
</configuration>
<configuration default="false" name="Unnamed" type="PythonConfigurationType" factoryName="Python">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="/usr/local/Cellar/python/2.7.13/Frameworks/Python.framework/Versions/2.7/bin/python2.7" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="cropper" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/cropper.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<method />
</configuration>
<list size="1">
<item index="0" class="java.lang.String" itemvalue="Python.Unnamed" />
</list>
</component>
<component name="ShelveChangesManager" show_recycled="false">
<option name="remove_strategy" value="false" />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="97259491-12e8-4b21-8042-1c50a8b8bff0" name="Default" comment="" />
<created>1494998403787</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1494998403787</updated>
</task>
<servers />
</component>
<component name="ToolWindowManager">
<frame x="22" y="223" width="1806" height="971" extended-state="0" />
<editor active="true" />
<layout>
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.2585034" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32942554" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="false" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
<window_info id="Data View" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
<window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" />
<window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
</layout>
</component>
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager />
<watches-manager />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/cropper.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding>
<element signature="e#0#21#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/cropper.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding>
<element signature="e#0#21#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/cropper.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="300">
<caret line="20" column="0" lean-forward="false" selection-start-line="20" selection-start-column="0" selection-end-line="20" selection-end-column="0" />
<folding>
<element signature="e#0#21#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</component>
</project>
Raw
utilities_cropper_cropper.py
from PIL import Image
import glob
import os
def make_dirs(p):
"""Create any needed directories in the target."""
path, file = os.path.split(p)
if os.path.exists(path): return
os.makedirs(path)
return
for imgpath in glob.glob('/Volumes/Seagate Backup Plus Drive/2017-05/*/*.jpg'):
img = Image.open(imgpath)
img2 = img.crop((0, 0, int(300*8.5), int(300*11)))
splitpath = imgpath.split(os.path.sep)
outpath = os.path.join('.', splitpath[-2], splitpath[-1])
make_dirs(outpath)
img2.save(outpath)
print outpath
Raw
utilities_ocr_corrections_.idea_misc.xml
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7.13 (/usr/local/Cellar/python/2.7.13/Frameworks/Python.framework/Versions/2.7/bin/python2.7)" project-jdk-type="Python SDK" />
</project>
Raw
utilities_ocr_corrections_.idea_modules.xml
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/ocr_corrections.iml" filepath="$PROJECT_DIR$/.idea/ocr_corrections.iml" />
</modules>
</component>
</project>
Raw
utilities_ocr_corrections_.idea_ocr_corrections.iml
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="Unittests" />
</component>
</module>
Raw
utilities_ocr_corrections_.idea_workspace.xml
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="56d04707-8e4b-44e1-af6e-acf145f24ee7" name="Default" comment="" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="TRACKING_ENABLED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="FileEditorManager">
<leaf>
<file leaf-file-name="ocrtransforms.py" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/ocrtransforms.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-694">
<caret line="180" column="22" lean-forward="true" selection-start-line="180" selection-start-column="22" selection-end-line="180" selection-end-column="22" />
<folding>
<element signature="e#0#9#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="re.py" pinned="false" current-in-tab="false">
<entry file="file:///usr/local/Cellar/python/2.7.13/Frameworks/Python.framework/Versions/2.7/lib/python2.7/re.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="326">
<caret line="154" column="0" lean-forward="false" selection-start-line="154" selection-start-column="0" selection-end-line="154" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Python Script" />
</list>
</option>
</component>
<component name="FindInProjectRecents">
<findStrings>
<find>marij</find>
<find>two</find>
<find>/</find>
<find>TWO</find>
<find>tabchar</find>
<find>w</find>
<find>***</find>
<find>PRi t</find>
</findStrings>
<replaceStrings>
<replace>r'\t'</replace>
<replace>word</replace>
</replaceStrings>
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/ocrtransforms.py" />
</list>
</option>
</component>
<component name="ProjectFrameBounds">
<option name="x" value="240" />
<option name="y" value="22" />
<option name="width" value="1680" />
<option name="height" value="1173" />
</component>
<component name="ProjectView">
<navigator currentView="ProjectPane" proportions="" version="1">
<flattenPackages />
<showMembers />
<showModules />
<showLibraryContents />
<hideEmptyPackages />
<abbreviatePackageNames />
<autoscrollToSource />
<autoscrollFromSource />
<sortByType />
<manualOrder />
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="Scope" />
<pane id="ProjectPane">
<subPane>
<expand>
<path>
<item name="ocr_corrections" type="b2602c69:ProjectViewProjectNode" />
<item name="ocr_corrections" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
<pane id="Scratches" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="settings.editor.selected.configurable" value="fileTemplates" />
<property name="SearchEverywhereHistoryKey" value="&#9;FILE&#9;file:///usr/local/Cellar/python/2.7.13/Frameworks/Python.framework/Versions/2.7/lib/python2.7/re.py" />
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="RunManager" selected="Python.ocrtransforms">
<configuration name="ocrtransforms" type="PythonConfigurationType" factoryName="Python" temporary="true">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="ocr_corrections" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/ocrtransforms.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
</configuration>
<recent_temporary>
<list size="1">
<item index="0" class="java.lang.String" itemvalue="Python.ocrtransforms" />
</list>
</recent_temporary>
</component>
<component name="ShelveChangesManager" show_recycled="false">
<option name="remove_strategy" value="false" />
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="56d04707-8e4b-44e1-af6e-acf145f24ee7" name="Default" comment="" />
<created>1504306076470</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1504306076470</updated>
</task>
<servers />
</component>
<component name="ToolWindowManager">
<frame x="240" y="22" width="1680" height="1173" extended-state="0" />
<editor active="true" />
<layout>
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.21306472" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.39905214" sideWeight="0.5081502" order="-1" side_tool="true" content_ui="tabs" />
<window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32985783" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="false" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32985783" sideWeight="0.4946846" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4815166" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="true" content_ui="tabs" />
<window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.39905214" sideWeight="0.49184975" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Data View" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" />
<window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
</layout>
</component>
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
<breakpoints-dialog>
<breakpoints-dialog />
</breakpoints-dialog>
<default-breakpoints>
<breakpoint type="python-exception">
<properties notifyOnTerminate="true" exception="BaseException">
<option name="notifyOnTerminate" value="true" />
</properties>
</breakpoint>
</default-breakpoints>
<option name="time" value="105" />
</breakpoint-manager>
<watches-manager />
</component>
<component name="editorHistoryManager">
<entry file="file://$APPLICATION_HOME_DIR$/helpers/pydev/pydevd.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="202">
<caret line="1022" column="0" lean-forward="false" selection-start-line="1022" selection-start-column="0" selection-end-line="1022" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file:///usr/local/Cellar/python/2.7.13/Frameworks/Python.framework/Versions/2.7/lib/python2.7/re.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="326">
<caret line="154" column="0" lean-forward="false" selection-start-line="154" selection-start-column="0" selection-end-line="154" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/ocrtransforms.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-694">
<caret line="180" column="22" lean-forward="true" selection-start-line="180" selection-start-column="22" selection-end-line="180" selection-end-column="22" />
<folding>
<element signature="e#0#9#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</component>
</project>
Raw
utilities_ocr_corrections_ocrtransforms.py
import re
import csv
fn = '/Users/Wes/NotForTheCloud/contests.txt'
ofn = '/Users/Wes/NotForTheCloud/OCRC_contests.csv'
# TODO: some of these look like they should already be handled
# LOW NCOME HOUSING UNITS
# /IMOBlLE HOME
# MOBELE HOME P
# MOBILE HOME P
# MED CAL
# MED CAL
# MED -CAL
# MEDl\tCAL
# PROPOS TlON
# /ICARRYOUT BAGS CHARGES
# CARRYOUT BAGS CHARGES
# /ICARRYOUT BAGS CHARGES
# CARRYOUT BAGS CHARGES
# BAN ON S NGLE U
# BAN ON SINGLE-USE
# BAN ON SlNGLE-USE
# /NOTE FOR
# VOTE FOR
class lister_dict(dict):
def __init__(self):
dict.__init__(self)
def __setitem__(self, key, value):
if key in self: self[key].add(value)
else:
s = set()
s.add(value)
dict.__setitem__(self, key, s)
class freq_dict(dict):
def __init__(self):
dict.__init__(self)
def count_item(self, key):
if key in self: self[key] += 1
else:
self[key]=1
x = 1 # target for breakpoint
class Wordfixer(dict):
def __init__(self):
dict.__init__(self)
def add(self, item):
if item in self:
self[item].count += 1
else:
t = self[item]
t.count = 1
t.score = None
def slurp(self, list):
''' Accept an enumeration of strings and create word entries. '''
for s in list:
for w in s.split():
self.add(w)
class OCRvaluemapper(dict):
''' A dictionary where the key is an input value straight from OCR
and the value is the corrected interpretation of the string. '''
def __init__(self, iterableoflists):
'''Initially everything is mapped to itself and dupes are removed.'''
dict.__init__(self)
for x in iterableoflists:
inv = x[0].strip()
self[inv] = inv
self.wfix = Wordfixer()
self.allcapletters = re.compile(r'^[A-Z]+$')
self.o_ir_i = re.compile(r'[OI]')
self.char_res = [(re.compile(x), y) for x, y in [
(r'^"(.+)"$', r'\1'),
(r'_+/?I', r'\t'),
(r'_+/?_', r'\t'),
(r'_', r' '),
(r' +', r' '),
(r'^l', ''),
(r'^I',''),
(r'^\s+', ''),
(r'^\s*\\\t', ''),
(r' *\t *', r'\t'),
(r'^ +', ''),
(r'\b[il1]', 'I'),
(r'[ \t]+$', '')
]]
self._character_edits()
self._word_edits()
self._leading_I_hack()
return
def mywords(self):
'''Iterator for all the words in all the current lines'''
for curline in self.values():
for word in curline.split():
yield word
def myuniquewords(self):
'''Iterator for all the unique words in the current lines'''
s = set([x for x in self.mywords()])
for item in s:
yield s
def _character_edits(self):
'''Fix character-based errors.'''
for refind, rep in self.char_res:
for currline, origline in [(self[k], k) for k in self.keys()]:
newl = re.sub(refind, rep, currline)
if newl <> currline:
self[origline] = newl
def _word_edits(self):
'''Fix word-based errors.'''
for subre, repl in self._get_fixers():
for k in self.keys():
currline, origline = self[k], k
newl = subre.sub(repl, currline)
if newl != currline:
self[origline] = newl
def output(self):
return [[key, self[key]] for key in self.keys()]
def _get_fixers(self):
''' Return a list of re-pairs which will fix certain
common OCR errors. For example, the world PROPOSITION will generate
replace 'PR[Oo0 ]P[Oo0 ]S[Ii|1]T[Ii|1][Oo0 ]N'
with 'PROPOSITION'.
Replacements are generated for all words that are found in all caps
that are at least four characters long.
'''
wordfixers = {}
for curline in self.values():
for word in curline.split():
x = curline.split()
if self.allcapletters.match(word) and self.o_ir_i.search(word) \
and len(word) >= 4:
refind = r'\b'
for c in word:
if c == 'I':
refind += r'[ \diI]'
elif c == 'O':
refind += '[ 0oO]'
else:
refind += c
refind += r'\b'
r = re.compile(refind)
wordfixers[r] = (r, word)
t = sorted([wordfixers[k] for k in wordfixers.keys()], key = lambda x: -len(x[1]))
return t
# def nearnuff(self, candidate, model):
# ''' Return true iff the candidate string is "near enough"
# to the model'''
#
# SUBSTRMIN = 4
# matchcount = 0
# comparecount = 0
# if len(model) <= SUBSTRMIN or len(candidate) <= SUBSTRMIN:
# raise Exception('operands must have at least {} characters'.format(SUBSTRMIN))
# model_substrs = []
# for substrlen in range(len(model) - 1, SUBSTRMIN - 1, -1):
# for substrstart in range(len(model) - substrlen + 1):
# substr = model[substrstart:substrstart + substrlen - 1]
# for candstart in range(0, len(candidate) - len(substr) + 1):
# print substr, candstart, len(substr), candidate[candstart: candstart + len(substr)]
# comparecount += 1
# if substr == candidate[candstart: candstart + len(substr)]:
# matchcount += 1
# return comparecount/float(matchcount)
# for (a,b) in [('abcde', 'abcdf'), ('abcdefg', 'abcxefg'), ('abcdef', 'abcde')]:
# x = t.nearnuff(a, b)
# print a, b, x
# exit(0)
# TODO: __mispelling_by_freq
# def __mispelling_by_freq(self):
# '''Find low frequency words similar to higher freq words and replace'''
#
# freqs = freq_dict()
#
# MIN_WORD_LENGTH = 4
#
# for line in self.values():
# for word in line.split():
# if len(word) >= MIN_WORD_LENGTH and word.isupper() and word.isalpha():
# freqs.count_item(word)
# for word in freqs.keys():
# if freqs[word] < 2:
# searches = []
def _leading_I_hack(self):
''' Other rules will delete a valid leading I in the word. This will
add wordfixers to try to put it back.'''
MIN_WORD_LENGTH = 4
words = lister_dict()
for line in self.values():
for word in line.split():
if len(word) >= MIN_WORD_LENGTH and word.isupper() and word.isalpha():
words[len(word)] = word
fixlist = []
for wordlen in sorted(words.keys(), key=lambda x: -x):
if wordlen < MIN_WORD_LENGTH + 1: break
for word in words[wordlen]:
if word.startswith('I'):
if word[1:] in words[wordlen - 1]:
fixlist.append(word)
for fixitem in fixlist:
pattern = r'\b' + fixitem[1:] + r'\b'
for (origline, curline) in [(k, self[k]) for k in self.keys()]:
newl = re.sub(pattern, fixitem, curline)
if newl != curline:
self[origline] = newl
# TODO: get number to vote for from OCR
# numword = ('TWO', 'THREE', 'FOUR', 'FIVE', 'SIX', 'SEVEN')
# votefors = {}
# for k in changes.keys():
# votefors[k] = 1
# for i in range(len(numword)):
# m = re.search(numword[i]+'( PARTY)?$', changes[k])
# if m:
# votefors[k]= i + 2
if __name__=='__main__':
with open(fn, 'r') as f:
csvin = csv.reader(f)
csvheader = csvin.next()
ovmap = OCRvaluemapper(csvin)
with open(ofn, 'w') as f:
csvout = csv.writer(f)
csvout.writerow(['voteop_good', 'voteop_other'])
outputs = []
for k in ovmap.keys():
outputs.append([ovmap[k], k])
outputs.sort()
s = set()
for outitem in outputs:
# p = lambda x: '>'.join(x.split('\t'))
# s.add(outitem[0])
# print p(outitem[0])
# print p(outitem[1])
# print
csvout.writerow(outitem)
# l =sorted( list(s))
# for i in l:
# print i
Raw
utilities_orgbypct_.idea_misc.xml
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7.13 (/usr/local/Cellar/python/2.7.13/Frameworks/Python.framework/Versions/2.7/bin/python2.7)" project-jdk-type="Python SDK" />
</project>
Raw
utilities_orgbypct_.idea_modules.xml
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/orgbypct.iml" filepath="$PROJECT_DIR$/.idea/orgbypct.iml" />
</modules>
</component>
</project>
Raw
utilities_orgbypct_.idea_orgbypct.iml
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="Unittests" />
</component>
</module>
Raw
utilities_orgbypct_.idea_workspace.xml
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="6e40b1ae-b750-4c79-9cde-fb6ace84595b" name="Default" comment="" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="TRACKING_ENABLED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="ExecutionTargetManager" SELECTED_TARGET="default_target" />
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file leaf-file-name="ballot_by_precinct.py" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/ballot_by_precinct.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="696">
<caret line="68" column="17" lean-forward="true" selection-start-line="67" selection-start-column="0" selection-end-line="68" selection-end-column="17" />
<folding>
<element signature="e#281#318#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="path.py" pinned="false" current-in-tab="false">
<entry file="file://$APPLICATION_HOME_DIR$/helpers/python-skeletons/os/path.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="120">
<caret line="9" column="0" lean-forward="false" selection-start-line="9" selection-start-column="0" selection-end-line="9" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="copy_images.ini" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/copy_images.ini">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="75">
<caret line="5" column="14" lean-forward="false" selection-start-line="5" selection-start-column="14" selection-end-line="5" selection-end-column="14" />
<folding />
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Python Script" />
</list>
</option>
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/database.ini" />
<option value="$PROJECT_DIR$/connect.py" />
<option value="$PROJECT_DIR$/config.py" />
<option value="$PROJECT_DIR$/pairs_to_dict.py" />
<option value="$PROJECT_DIR$/copy_images.ini" />
<option value="$PROJECT_DIR$/ballot_by_precinct.py" />
</list>
</option>
</component>
<component name="ProjectFrameBounds">
<option name="x" value="156" />
<option name="y" value="280" />
<option name="width" value="1388" />
<option name="height" value="971" />
</component>
<component name="ProjectView">
<navigator currentView="ProjectPane" proportions="" version="1">
<flattenPackages />
<showMembers />
<showModules />
<showLibraryContents />
<hideEmptyPackages />
<abbreviatePackageNames />
<autoscrollToSource />
<autoscrollFromSource />
<sortByType />
<manualOrder />
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="Scope" />
<pane id="Scratches" />
<pane id="ProjectPane">
<subPane>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="orgbypct" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="External Libraries" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ExternalLibrariesNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="orgbypct" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="orgbypct" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
</subPane>
</pane>
</panes>
</component>
<component name="PropertiesComponent">
<property name="settings.editor.selected.configurable" value="configurable.group.appearance" />
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="RunManager" selected="Python.ballot_by_precinct">
<configuration default="true" type="PythonConfigurationType" factoryName="Python">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="orgbypct" />
<option name="SCRIPT_NAME" value="" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<method />
</configuration>
<configuration default="true" type="Tox" factoryName="Tox">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="orgbypct" />
<method />
</configuration>
<configuration default="true" type="tests" factoryName="Doctests">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="orgbypct" />
<option name="SCRIPT_NAME" value="" />
<option name="CLASS_NAME" value="" />
<option name="METHOD_NAME" value="" />
<option name="FOLDER_NAME" value="" />
<option name="TEST_TYPE" value="TEST_SCRIPT" />
<option name="PATTERN" value="" />
<option name="USE_PATTERN" value="false" />
<method />
</configuration>
<configuration default="true" type="tests" factoryName="Unittests">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="orgbypct" />
<option name="_new_additionalArguments" value="&quot;&quot;" />
<option name="_new_target" value="&quot;.&quot;" />
<option name="_new_targetType" value="&quot;PATH&quot;" />
<method />
</configuration>
<configuration default="false" name="ballot_by_precinct" type="PythonConfigurationType" factoryName="Python">
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="/usr/local/Cellar/python/2.7.13/Frameworks/Python.framework/Versions/2.7/bin/python2.7" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<module name="orgbypct" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/ballot_by_precinct.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<method />
</configuration>
<list size="1">
<item index="0" class="java.lang.String" itemvalue="Python.ballot_by_precinct" />
</list>
</component>
<component name="ShelveChangesManager" show_recycled="false">
<option name="remove_strategy" value="false" />
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="6e40b1ae-b750-4c79-9cde-fb6ace84595b" name="Default" comment="" />
<created>1495224188891</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1495224188891</updated>
</task>
<servers />
</component>
<component name="ToolWindowManager">
<frame x="156" y="280" width="1388" height="971" extended-state="0" />
<editor active="true" />
<layout>
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.3320951" sideWeight="0.5" order="1" side_tool="false" content_ui="combo" />
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="10" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="true" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32942554" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="false" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.39976552" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="true" content_ui="tabs" />
<window_info id="Data View" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="8" side_tool="false" content_ui="tabs" />
<window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
<window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="9" side_tool="false" content_ui="tabs" />
<window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="3" side_tool="false" content_ui="combo" />
<window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32942554" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
</layout>
</component>
<component name="UnknownFeatures">
<option featureType="com.intellij.fileTypeFactory" implementationName="*.ini" />
</component>
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
<option name="time" value="7" />
</breakpoint-manager>
<watches-manager />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/ballot_by_precinct.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="810">
<caret line="54" column="0" lean-forward="false" selection-start-line="54" selection-start-column="0" selection-end-line="54" selection-end-column="0" />
<folding>
<element signature="e#281#318#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/copy_images.ini">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="75">
<caret line="5" column="14" lean-forward="false" selection-start-line="5" selection-start-column="14" selection-end-line="5" selection-end-column="14" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$APPLICATION_HOME_DIR$/helpers/python-skeletons/os/path.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="120">
<caret line="9" column="0" lean-forward="true" selection-start-line="9" selection-start-column="0" selection-end-line="9" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/config.py" />
<entry file="file://$PROJECT_DIR$/pairs_to_dict.py" />
<entry file="file://$PROJECT_DIR$/connect.py" />
<entry file="file://$PROJECT_DIR$/copy_images.ini">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="75">
<caret line="5" column="14" lean-forward="false" selection-start-line="5" selection-start-column="14" selection-end-line="5" selection-end-column="14" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$APPLICATION_HOME_DIR$/helpers/python-skeletons/os/path.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="120">
<caret line="9" column="0" lean-forward="false" selection-start-line="9" selection-start-column="0" selection-end-line="9" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/ballot_by_precinct.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="696">
<caret line="68" column="17" lean-forward="true" selection-start-line="67" selection-start-column="0" selection-end-line="68" selection-end-column="17" />
<folding>
<element signature="e#281#318#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</component>
</project>
Raw
utilities_orgbypct_ballot_by_precinct.py
#!/usr/bin/python
"""Copy TEVS image files from source to target directories, eliminating source
subdirectories and creating new target subdirectories based on precincet
name.
Initialization file includes source and target directories and database
parameters."""
from configparser import ConfigParser
import os
import psycopg2
import shutil
imagesSQL = """
select distinct file1, hart_precinct_name
from ballots
join voteops using (ballot_id)
join t2h_precinct ON precinct_code_string = tevs_precinct_code_string
order by hart_precinct_name, file1;"""
def copy_images():
parser = ConfigParser()
parser.read('copy_images.ini')
dbparams = dict(parser.items("postgresql"))
iopaths = dict(parser.items("directories"))
ipath = iopaths["source"]
if not os.path.isdir(ipath):
raise Exception("Source directory not available: {}".format(ipath))
opath = iopaths["target"]
if not os.path.isdir(opath):
try:
os.mkdir(opath)
except Exception as e:
raise Exception("Cannot find or create target directory: ({})".format(e))
conn = None
try:
conn = psycopg2.connect(**dbparams)
cur = conn.cursor()
cur.execute(imagesSQL)
print("The number of images: ", cur.rowcount)
row = cur.fetchone()
while row is not None:
# unpack source subdires, pack target
parts = row[0].split(os.sep)
frompath = os.path.join(ipath, parts[-2], parts[-1])
topath = os.path.join(opath, row[1], parts[-1])
todir = os.path.join(opath, row[1])
if not os.path.isdir(todir):
os.mkdir(todir)
try:
shutil.copy2(frompath, topath)
except Exception as e: # disambiguate file copy exception from database error
raise Exception("Cannot copy to:{}\n({})".format(topath, e))
row = cur.fetchone()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
if __name__ == '__main__':
copy_images()
Raw
utilities_orgbypct_copy_images.ini
[directories]
source = /Users/Wes/Dropbox/Programming/ElectionTransparency/cropper
target = ./out
[postgresql]
host=localhost
database=tevs
user=Wes
password=
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment