fixup_registry.jl

import Pkg
import Pkg.TOML
import Pkg.Types: VersionRange, ≲
using UUIDs
using GitHub
using ProgressMeter


# Copied from Registrator.jl
#=====================#
struct RegistryData
    name::String
    uuid::UUID
    repo::Union{String, Nothing}
    description::Union{String, Nothing}
    packages::Dict{String, Dict{String, Any}}
    extra::Dict{String, Any}
end

function RegistryData(
    name::AbstractString,
    uuid::Union{UUID, AbstractString};
    repo::Union{AbstractString, Nothing}=nothing,
    description::Union{AbstractString, Nothing}=nothing,
    packages::Dict=Dict(),
    extras::Dict=Dict(),
)
    RegistryData(name, UUID(uuid), repo, description, packages, extras)
end

function Base.copy(reg::RegistryData)
    RegistryData(
        reg.name,
        reg.uuid,
        reg.repo,
        reg.description,
        deepcopy(reg.packages),
        deepcopy(reg.extra),
    )
end

function parse_registry(io::IO)
    data = TOML.parse(io)

    name = pop!(data, "name")
    uuid = pop!(data, "uuid")
    repo = pop!(data, "repo", nothing)
    description = pop!(data, "description", nothing)
    packages = pop!(data, "packages", Dict())

    RegistryData(name, UUID(uuid), repo, description, packages, data)
end

parse_registry(str::AbstractString) = open(parse_registry, str)

function TOML.print(io::IO, reg::RegistryData)
    println(io, "name = ", repr(reg.name))
    println(io, "uuid = ", repr(string(reg.uuid)))

    if reg.repo !== nothing
        println(io, "repo = ", repr(reg.repo))
    end

    if reg.description !== nothing
        # print long-form string if there are multiple lines
        if '\n' in reg.description
            print(io, """\n
                description = \"\"\"
                $(reg.description)\"\"\"
                """
            )
        else
            println(io, "description = ", repr(reg.description))
        end
    end

    for (k, v) in pairs(reg.extra)
        TOML.print(io, Dict(k => v), sorted=true)
    end

    println(io, "\n[packages]")
    for (uuid, data) in sort!(collect(reg.packages), by=first)
        print(io, uuid, " = { ")
        print(io, "name = ", repr(data["name"]))
        print(io, ", path = ", repr(data["path"]))
        println(io, " }")
    end

    nothing
end
#===============================================================#

struct Package
    uuid::UUID
    name::String
    path::String
    url::String
    versions::Dict{VersionNumber, Base.SHA1}
end
Package(uuid, name, path, url) = Package(uuid, name, path, url, Dict{VersionNumber, Base.SHA1}())


function remove_registry(filter, regpath::String; print_deletions=false)
    packages = Package[]
    reg = parse_registry(joinpath(regpath, "Registry.toml"))
    for (uuid, data) in reg.packages
        path = data["path"]
        pkgpath = joinpath(regpath, path)
        packagefile = joinpath(pkgpath, "Package.toml")
        if !isfile(packagefile)
            @warn "did not find $pkgpath in registry"
            continue
        end
        pkgfile = Pkg.TOML.parsefile(packagefile)
        url = pkgfile["repo"]
        push!(packages, Package(UUID(uuid), data["name"], path, url))
    end
    # Using the fact that there is only one uuid per name in General right now
    pkgs_completely_deleted = Set{String}()
    sort!(packages; by= x -> x.name)
    @showprogress for pkg in packages
        pkgpath = joinpath(regpath, pkg.path)
        compatfile = joinpath(pkgpath, "Compat.toml")
        depfile = joinpath(pkgpath, "Deps.toml")

        merge!(pkg.versions, Pkg.Operations.load_versions(pkgpath; include_yanked = true))

       # @show pkg.versions
        versions_to_delete = VersionNumber[]
        compats = nothing
        if isfile(compatfile)
            compats = Pkg.Compress.load(joinpath(pkgpath, "Compat.toml"))
        end
        deps = nothing
        if isfile(depfile)
            deps = Pkg.Compress.load(joinpath(pkgpath, "Deps.toml"))
        end

        versions_to_delete = filter(compats, deps, pkg)

        foreach(v -> delete!(pkg.versions, v), versions_to_delete)

        if isempty(pkg.versions)
            delete!(reg.packages, string(pkg.uuid))
            rm(pkgpath, force=true, recursive=true)
            push!(pkgs_completely_deleted, pkg.name)
            @info "Deleting $(pkg.name) completely..."
        else
            if !isempty(versions_to_delete)
                # Write out new versions
                versions_file = joinpath(pkgpath, "Versions.toml")
                open(versions_file, "w") do io
                    versions_string = Dict(string(v) => Dict{String,Any}("git-tree-sha1" => string(sha)) for (v, sha) in pkg.versions)
                    Pkg.TOML.print(io, versions_string; sorted=true, by=x->VersionNumber(x))
                end
                # Write out the new compressed versions
                for file in (compatfile, depfile)
                    if isfile(file)
                        data = Pkg.Compress.load(file)
                        foreach(v -> delete!(data, v), versions_to_delete)
                        Pkg.Compress.save(file, data)
                    end
                end
            end
        end
    end
    open(joinpath(regpath, "Registry.toml"), "w") do io
        TOML.print(io, reg)
    end
    return pkgs_completely_deleted
end

function cleanup_registry(regpath::String)
    @info "Removing versions that depend on a package deleted from General"

    stdlibs = readdir(Sys.STDLIB)
    pkgs_general_deleted = readlines("packages_deleted")
    pkgs_completely_deleted = remove_registry(regpath, print_deletions=true) do compats, deps, pkg
        versions_to_delete = VersionNumber[]
        deps == nothing && return versions_to_delete
        for (version, compat) in deps
            for (deppkg, uuid) in compat
                deppkg in stdlibs && continue
                if deppkg in pkgs_general_deleted
                    @info "Deleting $(pkg.name):$(version) because it depends on $deppkg which was completely removed"
                    push!(versions_to_delete, version)
                    break
                end
            end
        end
        return versions_to_delete
    end

    write("packagages_deleted_registry", join(sort(collect(pkgs_completely_deleted)), '\n'))
    return
end

function check_registry_consistency(regpath::String)
    reg_general = parse_registry(joinpath(homedir(), ".julia/registries/General/Registry.toml")) 
    reg_custom = parse_registry(joinpath(regpath, "Registry.toml"))

    packages = Package[]
    all_uuids = Set{UUID}()

    for reg in [reg_general, reg_custom]
        for (uuid, data) in reg.packages
            push!(all_uuids, UUID(uuid))
        end
    end
    for (uuid, data) in reg_custom.packages
        push!(packages, Package(UUID(uuid), data["name"], data["path"], ""))
        push!(all_uuids, UUID(uuid))
    end
    stdlibs = readdir(Sys.STDLIB)

    for pkg in packages
        pkgpath = joinpath(regpath, pkg.path)
        depfile = joinpath(pkgpath, "Deps.toml")
        if isfile(depfile)
            deps = Pkg.Compress.load(joinpath(pkgpath, "Deps.toml"))
            for (version, compat) in deps
                for (deppkg, uuid) in compat
                    deppkg in stdlibs && continue
                    if !(UUID(uuid) in all_uuids)
                        error("$(pkg.name) depends on unknown package with uuid $uuid")
                    end
                end
            end
        end
    end
end


registry = ARGS[1]
cleanup_registry(registry)
check_registry_consistency(registry)

packages_deleted

AMG
ASCIIPlots
ASTInterpreter
ASTInterpreter2
AWS
AWSEC2
AWSIAM
AWSLambda
AWSSDB
AWSSES
AWSSNS
AbbrvKW
AbstractDomains
AbstractTables
Accumulo
ActiveAppearanceModels
Actors
AffineTransforms
Akagera
AlgebraicDiffEq
AndorSIF
AnsiColor
AppConf
Arbiter
ArrayViews
AudioIO
Augmentor
Augur
AutoDiffSource
AutoTypeParameters
Autoreload
Avro
BIGUQ
BOSSArrays
BSplines
BackpropNeuralNet
BanditOpt
Bandits
BaseTestDeprecated
BaseTestNext
BayesianDataFusion
Bebop
Benchmark
BenchmarkLite
Bezier
BioArgParse
BioBridgeR
BioSeq
BiomolecularStructures
Biryani
Blocks
Bokeh
BooSTjl
BoundingBoxes
Box0
Brim
Brownian
BuildExecutable
CBOR
CLArrays
CLBLAS
CLFFT
COFF
COMTRADE
CQL
CRC32
CRF
CUBLAS
CUDA
CUDAnativelib
CUDArt
CUDNN
CUFFT
CURAND
CUSOLVER
CUSPARSE
CaesarLCMTypes
Calc
Calendar
Cartesian
CasaCore
Catalan
CauseMap
Celeste
CellularAutomata
ChainRecursive
ChaosCommunications
Chatter
Checkers
ChemicalKinetics
Chipmunk
CholmodSolve2
Church
CirruParser
Clarus
ClinicalTrialSampleSize
ClobberingReload
Clockwork
ClusterDicts
ClusterUtils
CmplxRoots
CoinOptServices
Color
CombinatorialBandits
CommonCrawl
Commutator
CompilerOptions
CompilerTools
ComposeDiff
CompressedSensing
ConfidenceWeighted
ConicIP
ConnectSDK
ConnectionPools
ConstructiveSolidGeometry
ContMechTensors
ContinuedFractions
ControlToolbox
Convertible
ConvexSwitch
CoreNLP
CoverageBase
Cpp
Crispulator
CrossDecomposition
Crypto
Curl
Curvelet
DBAPI
DPMeansClustering
DReal
DWARF
DataCubes
DataStreamsIntegrationTests
DataTables
DataValueArrays
DateParser
Dates
Datetime
Debug
DebuggerFramework
DeclarativePackages
DelaunayMeshes
DeterminantalPointProcesses
DeterministicPolicyGradient
Devectorize
DictFiles
DictUtils
DiffBase
DiffEqApproxFun
DiffEqPy
DiffLinearAlgebra
DiffWrappers
DigitalComm
Digits
DimensionalityReduction
DirichletProcessMixtures
DiscretePredictors
DiscriminantAnalysis
Distance
Docile
Docker
Dopri
DoubleDouble
Drawing
DriftDiffusion
DriftDiffusionPoissonSystems
Dtree
DynMultiply
DynaWAVE
DynamicDiscreteModels
DynamoDB
Dynare
EAGOBranchBound
EAGODomainReduction
EAGOIntervalArithmetic
EAGOParametricInterval
EAGOSmoothMcCormickGrad
EEG
EGM96
ELF
EasyPhys
EasyPkg
EchoJulia
EchogramPyPlot
EcologicalNetwork
EconDatasets
EconModels
Eglob
EllipticFEM
EmpiricalRiskMinimization
EmpiricalRisks
Ensemble
EnvelopedArrays
Equations
EscapeString
Escher
Etcd
EventHistory
Evolutionary
EvolvingGraphs
ExcelDataStreams
ExecutableSpecifications
ExperimentalAnalysis
Expr2LaTeX
ExpressionPatterns
ExpressionUtils
Extern
ExtremelyRandomizedTrees
FEHM
FEMCoupling
FEMDynamics
FEModels
FLAC
FMIndexes
FNVHash
FPTControl
FProfile
FWF
FaceDatasets
FactCheck
FactorModels
FancyDiagnostics
FastAnonymous
FastArrayOps
FastCombinations
FinancialMarkets
FiniteElementDiffEq
FiniteStateMachine
FirstOrderLogic
FixedPoint
FixedSizeArrays
FixedSizeDictionaries
Fixtures
FlexibleArrays
FrontierEfficiencyAnalysis
Fuji
FunctionalData
FunctionalDataUtils
FusionDirect
GLAbstraction
GLBooks
GLPlot
GLText
GLVisualize
GLWindow
GUITestRunner
GadflyDiff
Gallium
Gasp
GeneralizedMetropolisHastings
GeneralizedSampling
GeneralizedSchurAlgorithm
GeneticAlgorithms
GeoIP
GeographicLibPy
GibbsSeaWater
Gillespie
GitLab
GnuTLS
GoogleCharts
GradientBoost
Graft
GraphCentrality
GraphGLRM
GraphLayout
GraphMatrices
GraphViz
GreatCircle
Grid
GroveAlg
GtkBuilderAid
Gunrock
HDFS
HPAT
HPack
HSA
HTSLIB
HTTP2
HeadlessChromium
Helpme
HexEdit
HiddenMarkovModels
HigherOrderClustering
HigherPrecision
Hinton
Homotopies
Hop
HttpParser
HttpServer
ICOADSDict
ICU
IDRsSolver
IDXParser
IJuliaPortrayals
IOIndents
IPPCore
IPPDSP
IProfile
ISPC
ImageRegistration
ImmutableArrays
ImputeNaNs
IncGammaBeta
IndependentRandomSequences
IndexedArrays
InformedDifferentialEvolution
Instruments
IntModN
IntegerSmithNormalForm
InteractUIkit
Interfaces
IntervalLinearEquations
IntervalWavelets
InvariantCausal
IsoRank
Isotonic
IssueReporter
Iterators
Ito
J4Org
JACKAudio
JFVM
JLDArchives
JPLEphemeris
JUDI
JellyFish
Jewel
JointMoments
JuMPChance
JuliaParser
JulieTest
Julz
Jumos
KCores
KDTrees
KSVD
KShiftsClustering
Kafka
KernSmooth
Keys
Klara
LARS
LCA
LMDB
LTISystems
LaTeX
LazySequences
LempelZiv
Lexicon
LibArchive
LibBSON
LibCloud
LibGit2
LibHealpix
LibTrading
Liga
LineEdit
LinearAlgebraicRepresentation
LinearLeastSquares
LinearResponseVariationalBayes
LinguisticData
Lint
LocalMemoize
LoggedDicts
Logging
Lora
Loss
LowDimNearestNeighbors
Lumberjack
Lumira
LyapunovExponents
MAB
MATLABCluster
MCMC
MDPs
MNIST
MP3
MPFI
MRMissing
MUMPSjInv
MachO
MachineLearning
MachineLearningMetrics
Maker
Mandrill
ManualMemory
MapLight
MarketTechnicals
MarkovTransitionMatrices
MathToolkit
Mathematica
MatlabCompat
MatpowerCases
MatrixChainMultiply
Mayday
Meddle
MelGeneralizedCepstrums
MergeSorted
MergedMethods
Merlin
Meshes
MessageUtils
MetaTools
MetadataTools
Metamath
MichiganAutonomousVehicles
Miniball
MinimalPerfectHashes
MixtureModels
Mocha
ModelReduction
MolecularDataType
MolecularDynamics
MolecularPDB
Mongo
Morsel
Mortar2D
Mortar3D
MotoServer
MovingWeightedLeastSquares
MsgPackRpcServer
MuKanren
MultiNest
MultiPoly
MultidimensionalTables
Multirate
Murmur3
NBTesting
NEOS
NHST
NKLandscapes
NLOptControl
NMR
NOAAData
NURBS
NamedDimensions
Napier
NaturalSelection
NaturalSort
Neovim
NetalignMeasures
NetalignUtils
NetworkFlows
NetworkViz
NeuralynxNCS
NextGenSeqUtils
NodeNumbering
NonNegLeastSquares
NullableArrays
Nulls
NumFormat
NumberedLines
NumericExtensions
NumericFuns
NumericSuffixes
OCCA
OEIS
OSC
OSXNotifier
ObjFileBase
OpenFOAM
OpenGene
OpenSecrets
OpenSlide
OpenStreetMap
Options
Opus
Orchestra
PAINTER
PDFExtract
PEGParser
PGM
PLSRegressor
PLX
PSPlot
PValueAdjust
PackageStream
PageAlignedArrays
ParallelAccelerator
ParallelSparseMatMul
Pastebin
Patchwork
PatternDispatch
Pcap
PdbTool
Pedigrees
Perceptrons
Persist
PhantomJS
Phonetics
Phylogenetics
Phylogenies
PhysConsts
PiecewiseAffineTransforms
PiecewiseIncreasingRanges
PkgDev
PlanOut
Playground
PlotRecipes
PolarFact
PolyaGammaDistribution
Polyglot
Polynomial
PortAudio
PortableGameNotation
PowerLaws
PowerSeries
PrettyNumber
PrettyPlots
PrivateModules
PrivateMultiplicativeWeights
ProgressiveAligner
ProjectTemplate
Promisables
PropertyGraph
PublicSuffix
Push
PyAMG
PyLCM
PyLexYacc
PyLogging
PyProj
PySide
PySyntax
PyTest
PyX
Pyramids
Q
QDXML
QML
QPSParser
QRupdate
QWTwPlot
QuDirac
QuTiP
Quandl
QuantumBayesian
QuantumLab
QuantumTomography
Queueing
QuickHull
QuickShiftClustering
Qwt
RDF
REPL
REPLGameOfLife
ROOT
Rainflow
RandomFerns
Ranges
RationalExtensions
RationalFunctions
ReactiveBasics
ReadWriteLocks
RealInterface
RealSense
Reconstructables
Redis
RegERMs
Regression
Requests
RequestsCache
Restful
Revealables
ReverseDiffOverload
ReverseDiffSource
Revtok
Rif
Rifraf
RingArrays
RobustLeastSquares
RobustStats
RouletteWheels
RudeOil
RunTests
SALSA
SDE
SFML
SGDOptim
SIUnits
SLHA
SPTK
SQSChannels
SVM
SVMLightLoader
Samantha
Sampling
SaveREPL
SchroedingersSmoke
SearchMethods
SecureSessions
Seismic
SemidefiniteModel
SemidefiniteProgramming
SeqMaker
Shannon
ShapeModels
Shearlab
ShowItLikeYouBuildIt
Sigma
SigmoidalProgramming
SignalView
Silo
SimpleStructs
SimpleTasks
Sims
SloanDigitalSkySurvey
Slugify
Smile
SnFFT
SnowballStemmer
SoftConfidenceWeighted
SolarSystemLib
Somoclu
SortJoin
Sparklines
SparseInterp
SparseMatrixLILs
SparseVectors
SpatialGrids
SpeedDate
SpikingNetworks
StackTraces
StackedNets
StatefulIterators
Stats
StatsdClient
SteinDiscrepancy
StingerGraphs
StochasticSearch
Stochy
StokesDiffEq
StrPack
StreamStats
StringCases
StringInterpolation
StringLiterals
StringParserPEG
StructsOfArrays
StructuredQueries
Subsequences
SubsetSelection
Sugar
SunlightAPIs
Surprise
SwiftObjectStores
Swifter
Switch
SwitchTimeOpt
SymmetricBitArrays
Synchrony
SynthesisFilters
SystemImageBuilder
TOML
TablesDemo
TakingBroadcastSeriously
TermWin
TerminalUI
Terminals
TestReports
TestRunner
TestSetExtensions
Tetris
TexExtensions
TextModel
TextPlots
ThermodynamicsTable
ThingSpeak
ThreeJS
TimeData
TimeModels
TimeSeriesIO
Timestamps
TinyInt
TinySparse
Tinyurl
Tk
Tokamak
TopicModels
TraceCalls
Transit
Transpiler
TwoBasedIndexing
TypeCheck
Typeclass
URITemplate
UTF16
UUID
Units
UnsafeAtomics
Unums
VLFeat
VML
VStatistic
ValueDispatch
ValueSymbols
VarianceComponentTest
Vectorize
Vega
VehicleModels
VennEuler
Viridis
VirtualArrays
Voronoi
Voting
Vue
WCSLIB
Wallace
Watcher
WaveletMatrices
WebFuncs
Weber
WeberCedrus
WeberDAQmx
Whippet
Whittaker
XClipboard
XDiff
XKPasswd
XMLRPC
XMLconvert
XSV
YT
Yelp
ZipCode
Zlib
fstformat
mPulseAPI