Hex to binary.ipynb

{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"from pyspark.sql import SparkSession\n\nspark = SparkSession.builder.getOrCreate()","execution_count":1,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = spark.createDataFrame([('0x414243',)], ['a'])","execution_count":2,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.createOrReplaceTempView(\"test\")\nspark.sql(\"SELECT * FROM test\").show()","execution_count":3,"outputs":[{"output_type":"stream","text":"+--------+\n|       a|\n+--------+\n|0x414243|\n+--------+\n\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"spark.sql(\"SELECT UNHEX(SUBSTRING(a, 3)) FROM test\").show()","execution_count":4,"outputs":[{"output_type":"stream","text":"+----------------------------------+\n|unhex(substring(a, 3, 2147483647))|\n+----------------------------------+\n|                        [41 42 43]|\n+----------------------------------+\n\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"The cell above shows that we convert the hex literal into bytes.\nThe cell below casts this to a string to show that this works, but that's not what we need for actual binary data like `TX_HASH`."},{"metadata":{"trusted":true},"cell_type":"code","source":"spark.sql(\"SELECT CAST(UNHEX(SUBSTRING(a, 3)) AS string) FROM test\").show()","execution_count":5,"outputs":[{"output_type":"stream","text":"+--------------------------------------------------+\n|CAST(unhex(substring(a, 3, 2147483647)) AS STRING)|\n+--------------------------------------------------+\n|                                               ABC|\n+--------------------------------------------------+\n\n","name":"stdout"}]}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3 (ipykernel)","language":"python"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":5}

requirements.txt

pyspark