Skip to content

Instantly share code, notes, and snippets.

@fonsp
Created March 7, 2022 20:26
Show Gist options
  • Save fonsp/d2b84265012942dc40d0082b1fd405ba to your computer and use it in GitHub Desktop.
Save fonsp/d2b84265012942dc40d0082b1fd405ba to your computer and use it in GitHub Desktop.
JS and Julia implementations for base64url, a non-standard variant of base64. See https://en.wikipedia.org/wiki/Base64#Variants_summary_table
### A Pluto.jl notebook ###
# v0.18.1
using Markdown
using InteractiveUtils
# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
quote
local iv = try Base.loaded_modules[Base.PkgId(Base.UUID("6e696c72-6542-2067-7265-42206c756150"), "AbstractPlutoDingetjes")].Bonds.initial_value catch; b -> missing; end
local el = $(esc(element))
global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : iv(el)
el
end
end
# ╔═╡ 06c24cda-998d-4909-8ec7-4699eb9abd3d
using BenchmarkTools
# ╔═╡ f097d5d8-fcd2-44bc-92b9-263e0361eae8
using Base64
# ╔═╡ 0f5f4f39-dd61-465d-a1d8-d4329cd49977
using PlutoUI
# ╔═╡ f228aa17-35e2-4a04-babb-488c8fc88342
const data = rand(UInt8, 10000)
# ╔═╡ 42b680f0-c101-4197-b1ad-02b10889edff
const b64 = base64encode(data)
# ╔═╡ d1142d74-f064-4d7d-9f27-5cb96b163e5c
function d1(s::AbstractString)
base64decode(replace(replace(s, '-' => '+'), '_' => '/'))
end
# ╔═╡ ff2b07c2-00ee-429f-bcca-2377118e684f
function e1(data)
replace(replace(base64encode(data), '+' => '-'), '/' => '_')
end
# ╔═╡ 67329efe-8005-4126-9e9a-f46b6af8a266
# ╔═╡ faa7b671-87a7-445d-b44e-3800fe10a7ed
const plus = codeunit("+", 1)
# ╔═╡ 73af90e3-c3db-4bc1-90a2-13c93291d438
const minus = codeunit("-", 1)
# ╔═╡ 800f4fea-44e2-4a27-a61b-f8691e433a8b
const slash = codeunit("/", 1)
# ╔═╡ 35956fe0-558d-40c8-a21c-e0b6634aff3c
const equals = codeunit("=", 1)
# ╔═╡ 2f63bdb1-8c88-4aeb-a8a5-85786840c7a2
const underscore = codeunit("_", 1)
# ╔═╡ 6995536f-c27d-40c7-9584-7e45f0def019
without_equals(s) = rstrip(s, '=') # is a lazy SubString! yay
# ╔═╡ 13b3ae1d-f99c-4428-b9cc-1db8b9f08a49
function e2(data)
String(replace(
base64encode(data) |> without_equals |> codeunits,
plus => minus, slash => underscore
))
end
# ╔═╡ bec7bb42-6d7a-4c0d-a098-ffd3f0418a7a
tdata = UInt8[0,0,63,0,0,62,42]
# ╔═╡ 81f6efee-ca2b-48c0-82a6-6f518e37f298
begin
struct P <: IO
io::IO
end
Base.isreadable(::P) = false
Base.iswritable(pipe::P) = iswritable(pipe.io)
function Base.unsafe_write(pipe::P, ptr::Ptr{UInt8}, n::UInt)::Int
for i in 1:n
x = unsafe_load(ptr, i)
Base.write(pipe.io, x === plus ? minus : x === slash ? underscore : x)
end
n
end
Base.write(pipe::P, x::UInt8) = x !== equals && Base.write(pipe.io, x === plus ? minus : x === slash ? underscore : x)
end
# ╔═╡ 70dad70c-043c-4fd8-ad24-7f0c925f30c8
function d2(s::AbstractString)
cs = codeunits(s)
io = IOBuffer(; sizehint=length(cs) + 2)
iob64_decode = Base64DecodePipe(io)
write(io,
replace(codeunits(s), minus => plus, underscore => slash)
)
for _ in 1:mod(-length(cs),4)
write(io, equals)
end
seekstart(io)
read(iob64_decode)
end
# ╔═╡ b52f08da-f0f7-4242-bdd2-9feb532713a5
let
io = IOBuffer()
iob64_encode = Base64EncodePipe(P(io))
write(iob64_encode, tdata)
close(iob64_encode)
String(take!(io))
end
# ╔═╡ 4e824569-557c-40aa-877a-0b71c3798dd6
function e3(data)
io = IOBuffer(; sizehint=((length(data) + 3) * 4) ÷ 3)
iob64_encode = Base64EncodePipe(P(io))
write(iob64_encode, data)
close(iob64_encode)
return String(take!(io))
end
# ╔═╡ 39e06ce3-e501-421f-a8f7-bbfc299ce8ee
function d3(s::AbstractString)
cs = codeunits(s)
io = IOBuffer(; sizehint=length(cs) + 2)
iob64_decode = Base64DecodePipe(io)
for c in cs
write(io,
if c === minus
plus
elseif c === underscore
slash
else
c
end
)
end
for _ in 1:mod(-length(cs),4)
write(io, equals)
end
seekstart(io)
read(iob64_decode)
end
# ╔═╡ 04fa5e14-9976-49e1-8c15-b56fa013235f
base64decode("asdf")
# ╔═╡ 097e2fa2-6dbe-472a-8ab8-5ebe45a6c052
d3("AQID")
# ╔═╡ 105a6b06-5896-4a83-9c0b-9e050133eb55
d3("AAAA")
# ╔═╡ 1c251eb6-ca4d-4645-8b8c-bba92d0355f4
@code_warntype e3(UInt8[1,2,3])
# ╔═╡ d4023091-5394-4790-8b3a-297d53b945ce
base64decode("AAA+AAA/ZMg")
# ╔═╡ 63ce464d-a054-404f-a637-92e066192788
# ╔═╡ cd8f4fe3-0621-45d1-a3f4-5ab5a8b08ab4
base64encode(UInt8[1,2,3,4,5])
# ╔═╡ 1e8751db-9636-4fde-8e26-7cedf1c59851
@bind x Slider(0:255)
# ╔═╡ 1944f0f3-82e5-43b1-b4af-0da5e5973fa4
const minidata = UInt8[0, 0, 63, 0, 0, 62, 100, 200]
# ╔═╡ 9dcc2f22-f650-4999-8958-6a9ebc1355b7
function bench(e,d)
e_bench = @benchmark $e($data) seconds = 3
b64u = e(data)
d_bench = @benchmark $d($b64u) seconds = 3
(; sample_e=e(minidata), sample_d=d(e(minidata)), e_bench, d_bench)
end
# ╔═╡ 249ed8b1-29df-4651-838d-2590d3665c1a
bench(e3, d3)
# ╔═╡ 6bcfc114-2387-46d0-b403-f026daff3884
bench(base64encode, base64decode)
# ╔═╡ 95c13505-c53c-43f8-beee-eb5d18f91a2e
bench(e2, d2)
# ╔═╡ e1745ee4-1c9e-471b-a163-3edfbe8202f8
bench(e1, d1)
# ╔═╡ 00000000-0000-0000-0000-000000000001
PLUTO_PROJECT_TOML_CONTENTS = """
[deps]
Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
PlutoUI = "7f904dfe-b85e-4ff6-b463-dae2292396a8"
[compat]
BenchmarkTools = "~1.3.1"
PlutoUI = "~0.7.36"
"""
# ╔═╡ 00000000-0000-0000-0000-000000000002
PLUTO_MANIFEST_TOML_CONTENTS = """
# This file is machine-generated - editing it directly is not advised
julia_version = "1.7.0"
manifest_format = "2.0"
[[deps.AbstractPlutoDingetjes]]
deps = ["Pkg"]
git-tree-sha1 = "8eaf9f1b4921132a4cff3f36a1d9ba923b14a481"
uuid = "6e696c72-6542-2067-7265-42206c756150"
version = "1.1.4"
[[deps.ArgTools]]
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
[[deps.Artifacts]]
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
[[deps.Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
[[deps.BenchmarkTools]]
deps = ["JSON", "Logging", "Printf", "Profile", "Statistics", "UUIDs"]
git-tree-sha1 = "4c10eee4af024676200bc7752e536f858c6b8f93"
uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
version = "1.3.1"
[[deps.ColorTypes]]
deps = ["FixedPointNumbers", "Random"]
git-tree-sha1 = "024fe24d83e4a5bf5fc80501a314ce0d1aa35597"
uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"
version = "0.11.0"
[[deps.CompilerSupportLibraries_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
[[deps.Dates]]
deps = ["Printf"]
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
[[deps.Downloads]]
deps = ["ArgTools", "LibCURL", "NetworkOptions"]
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
[[deps.FixedPointNumbers]]
deps = ["Statistics"]
git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc"
uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93"
version = "0.8.4"
[[deps.Hyperscript]]
deps = ["Test"]
git-tree-sha1 = "8d511d5b81240fc8e6802386302675bdf47737b9"
uuid = "47d2ed2b-36de-50cf-bf87-49c2cf4b8b91"
version = "0.0.4"
[[deps.HypertextLiteral]]
git-tree-sha1 = "2b078b5a615c6c0396c77810d92ee8c6f470d238"
uuid = "ac1192a8-f4b3-4bfe-ba22-af5b92cd3ab2"
version = "0.9.3"
[[deps.IOCapture]]
deps = ["Logging", "Random"]
git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a"
uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89"
version = "0.2.2"
[[deps.InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
[[deps.JSON]]
deps = ["Dates", "Mmap", "Parsers", "Unicode"]
git-tree-sha1 = "3c837543ddb02250ef42f4738347454f95079d4e"
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
version = "0.21.3"
[[deps.LibCURL]]
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
[[deps.LibCURL_jll]]
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
[[deps.LibGit2]]
deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
[[deps.LibSSH2_jll]]
deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
[[deps.Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
[[deps.LinearAlgebra]]
deps = ["Libdl", "libblastrampoline_jll"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
[[deps.Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
[[deps.Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
[[deps.MbedTLS_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
[[deps.Mmap]]
uuid = "a63ad114-7e13-5084-954f-fe012c677804"
[[deps.MozillaCACerts_jll]]
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
[[deps.NetworkOptions]]
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
[[deps.OpenBLAS_jll]]
deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
[[deps.Parsers]]
deps = ["Dates"]
git-tree-sha1 = "85b5da0fa43588c75bb1ff986493443f821c70b7"
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "2.2.3"
[[deps.Pkg]]
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
[[deps.PlutoUI]]
deps = ["AbstractPlutoDingetjes", "Base64", "ColorTypes", "Dates", "Hyperscript", "HypertextLiteral", "IOCapture", "InteractiveUtils", "JSON", "Logging", "Markdown", "Random", "Reexport", "UUIDs"]
git-tree-sha1 = "2c87c85e397b7ffed5ffec054f532d4edd05d901"
uuid = "7f904dfe-b85e-4ff6-b463-dae2292396a8"
version = "0.7.36"
[[deps.Printf]]
deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
[[deps.Profile]]
deps = ["Printf"]
uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"
[[deps.REPL]]
deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
[[deps.Random]]
deps = ["SHA", "Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
[[deps.Reexport]]
git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
version = "1.2.2"
[[deps.SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
[[deps.Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
[[deps.Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
[[deps.SparseArrays]]
deps = ["LinearAlgebra", "Random"]
uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
[[deps.Statistics]]
deps = ["LinearAlgebra", "SparseArrays"]
uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
[[deps.TOML]]
deps = ["Dates"]
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
[[deps.Tar]]
deps = ["ArgTools", "SHA"]
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
[[deps.Test]]
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
[[deps.UUIDs]]
deps = ["Random", "SHA"]
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
[[deps.Unicode]]
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
[[deps.Zlib_jll]]
deps = ["Libdl"]
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
[[deps.libblastrampoline_jll]]
deps = ["Artifacts", "Libdl", "OpenBLAS_jll"]
uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
[[deps.nghttp2_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
[[deps.p7zip_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
"""
# ╔═╡ Cell order:
# ╠═06c24cda-998d-4909-8ec7-4699eb9abd3d
# ╠═f097d5d8-fcd2-44bc-92b9-263e0361eae8
# ╠═f228aa17-35e2-4a04-babb-488c8fc88342
# ╠═42b680f0-c101-4197-b1ad-02b10889edff
# ╠═d1142d74-f064-4d7d-9f27-5cb96b163e5c
# ╠═ff2b07c2-00ee-429f-bcca-2377118e684f
# ╠═67329efe-8005-4126-9e9a-f46b6af8a266
# ╠═faa7b671-87a7-445d-b44e-3800fe10a7ed
# ╠═73af90e3-c3db-4bc1-90a2-13c93291d438
# ╠═800f4fea-44e2-4a27-a61b-f8691e433a8b
# ╠═35956fe0-558d-40c8-a21c-e0b6634aff3c
# ╠═2f63bdb1-8c88-4aeb-a8a5-85786840c7a2
# ╠═70dad70c-043c-4fd8-ad24-7f0c925f30c8
# ╠═13b3ae1d-f99c-4428-b9cc-1db8b9f08a49
# ╠═6995536f-c27d-40c7-9584-7e45f0def019
# ╠═bec7bb42-6d7a-4c0d-a098-ffd3f0418a7a
# ╠═81f6efee-ca2b-48c0-82a6-6f518e37f298
# ╠═b52f08da-f0f7-4242-bdd2-9feb532713a5
# ╠═4e824569-557c-40aa-877a-0b71c3798dd6
# ╠═39e06ce3-e501-421f-a8f7-bbfc299ce8ee
# ╠═04fa5e14-9976-49e1-8c15-b56fa013235f
# ╠═097e2fa2-6dbe-472a-8ab8-5ebe45a6c052
# ╠═105a6b06-5896-4a83-9c0b-9e050133eb55
# ╠═1c251eb6-ca4d-4645-8b8c-bba92d0355f4
# ╠═249ed8b1-29df-4651-838d-2590d3665c1a
# ╠═6bcfc114-2387-46d0-b403-f026daff3884
# ╠═95c13505-c53c-43f8-beee-eb5d18f91a2e
# ╠═e1745ee4-1c9e-471b-a163-3edfbe8202f8
# ╠═d4023091-5394-4790-8b3a-297d53b945ce
# ╠═63ce464d-a054-404f-a637-92e066192788
# ╠═cd8f4fe3-0621-45d1-a3f4-5ab5a8b08ab4
# ╠═0f5f4f39-dd61-465d-a1d8-d4329cd49977
# ╠═1e8751db-9636-4fde-8e26-7cedf1c59851
# ╠═1944f0f3-82e5-43b1-b4af-0da5e5973fa4
# ╠═9dcc2f22-f650-4999-8958-6a9ebc1355b7
# ╟─00000000-0000-0000-0000-000000000001
# ╟─00000000-0000-0000-0000-000000000002
const base64_arraybuffer = async (data) => {
// Use a FileReader to generate a base64 data URI
const base64url = await new Promise((r) => {
const reader = new FileReader()
reader.onload = () => r(reader.result)
reader.readAsDataURL(new Blob([data]))
})
/*
The result looks like
"data:application/octet-stream;base64,<your base64 data>",
so we split off the beginning:
*/
return base64url.split(",", 2)[1]
}
const base64url_arraybuffer1 = async (data) => {
let original = await base64_arraybuffer(data)
return original.replaceAll("+", "-").replaceAll("/", "_").replace(/\=+$/, "")
}
// A single regex, with a function to replace
const base64url_arraybuffer2 = async (data) => {
let original = await base64_arraybuffer(data)
return original.replaceAll(/[\+\/\=]/g, (s) => (s === "+" ? "-" : s === "/" ? "_" : ""))
}
const base64url_arraybuffer2a = async (data) => {
let original = await base64_arraybuffer(data)
return original.replaceAll(/[\+\/\=]/g, (s) => {
const c = s.charCodeAt(0)
return c === 43 ? "-" : c === 47 ? "_" : ""
})
}
// Same as last one, but the = trimming at the end is not handled by the regex, but manually.
const base64url_arraybuffer3 = async (data) => {
let original = await base64_arraybuffer(data)
const almost = original.replaceAll(/[\+\/]/g, (s) => (s === "+" ? "-" : "_"))
if (almost.length < 2) {
return almost
} else {
const offset = almost[almost.length - 1] === "=" ? (almost[almost.length - 2] === "=" ? 2 : 1) : 0
return almost.slice(0, almost.length - offset)
}
}
const enc = new TextEncoder()
const dec = new TextDecoder()
const plus = enc.encode("+")[0]
const slash = enc.encode("/")[0]
const equals = enc.encode("=")[0]
const dash = enc.encode("-")[0]
const underscore = enc.encode("_")[0]
const base64url_arraybuffer4 = async (data) => {
const original = await base64_arraybuffer(data)
const buffer = enc.encode(original)
for (let i = 0; i < buffer.length; i++) {
const old = buffer[i]
buffer[i] = old === plus ? dash : old === slash ? underscore : old
}
return dec.decode(buffer)
}
let measure = async (f) => {
// create some random data in a small buffer
let smallbuffer = new Uint8Array(50000)
crypto.getRandomValues(smallbuffer)
// create a large buffer (10MB) with this random data
let length = 10000000
let buffer = new Uint8Array(length)
buffer.forEach((_, i) => {
buffer[i] = smallbuffer[i % 50000]
})
// call our function
let start = performance.now()
await f(buffer)
let stop = performance.now()
// return a measurement
let seconds = (stop - start) / 1000
let megabytes = length / 1000 / 1000
return `MB per second: ${Math.floor(megabytes / seconds)}`
}
// example use:
for (let f of [base64_arraybuffer, base64url_arraybuffer1, base64url_arraybuffer2, base64url_arraybuffer2a, base64url_arraybuffer3, base64url_arraybuffer4]) {
console.log(f.name)
console.log(await f(new Uint8Array([0, 0, 63, 0, 0, 62, 100, 200])))
for (let i of new Array(10)) {
console.log(await measure(f))
}
}
// returns: 'MB per second: 261'
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment