sketch of bits of incremental update and mirroring logic

gistfile1.hs

data RepositoryInfo = RepositoryInfo {
       repoName       :: String,
       repoPrimaryURL :: URL,
       repoMirrorURLs :: [URL],
       repoLocalDir   :: FilePath
       repoSettings   :: DownloadSettings
     }

data DownloadSettings = DownloadSettings {
       disableAllVerification        :: Bool,
       disableTimestampVerification  :: Bool,
       disableIncrementalUpdate      :: Bool,
       disableTransportCompression   :: Bool
     }

repoLocalIndexFileCompressed :: RepositoryInfo -> FilePath
repoLocalIndexFileCompressed
  RepositoryInfo { repoLocalDir } = repoLocalDir </> "00-index.tar.gz"

repoLocalIndexFileUncompressed :: RepositoryInfo -> FilePath
repoLocalIndexFileUncompressed
  RepositoryInfo { repoLocalDir } = repoLocalDir </> "00-index.tar"

updateRepositoryIndex :: Verbosity
                      -> RepositoryInfo
                      -> IO ()

updateRepositoryIndex verbosity repoinfo = do


    localIndexInfo <- checkLocalIndex repoinfo

    -- now try each mirror in turn
    tryMirrors [] (repoMirrorURLs repoinfo)

  where
    tryMirrors failures (mirror:mirrors) = do
      result <- updateIndexFromMirror
      case result of
        MirrorUnavailable -> do
          tryMirrors ((mirror, result):failures) mirrors

    tryMirrors failures [] = do
      -- construct and print/return summary report


updateIndexFromMirror :: URL -> 
updateIndexFromMirror mirrorURL =
    
    res <- downloadIndexManifest
    case res of
      Left  err
      Right manifestFile ->
        case verifyManifestFile manifestFile of
          
          Right manifestInfo ->
            case pickUpdateMethod of
              TryIncrementalUpdate
              TryFullUpdate
        
updateIndexFromMirrorIncremental = do
    let 
    res <- downloadFileRange expectedSize ( <//> ) localfile

updateIndexFromMirrorFull 
    res <- downloadFile expectedSize ( <//> ) localfile
    case res of
      Right _ -> do
        verifyHash hash localfile

data DownloadFailure = HostUnreachable
                     | ResourceUnavailable
                     | ResourceVerificationError

downloadIndexManifest :: URL -> m (Either () IndexManifestResponse)
downloadIndexManifest url = undefined
  

data IndexManifestResponse = IndexManifestResponse {
       indexManifestCompressedInfo   :: (Size, Hash),
       indexManifestUncompressedInfo :: Maybe (Size, Hash),
       indexClaimsAppendOnly         :: Bool,
       serverSupportsRangeRequests   :: Bool,
       serverSupportsGzipTransport   :: Bool,
     }

data UpdateMethod = TryIncrementalUpdate Size Size Hash
                  | TryFullUpdate             Size Hash

pickUpdateMethod :: DownloadSettings
                 -> LocalIndexFileInfo
                 -> IndexManifestResponse
                 -> UpdateMethod
pickUpdateMethod
    DownloadSettings {
       disableIncrementalUpdate = False,
       disableTransportCompression,
    }
    (HaveLocalIndexFile localUncompressesSize)
    IndexManifestResponse {
      indexClaimsAppendOnly         = True
      serverSupportsRangeRequests   = True,
      indexManifestUncompressedInfo = Just (remoteUncompressedSize, hash),
      indexManifestCompressedInfo   = (remoteCompressedSize, _),
    }
    | estimatedIncrementalUpdateSize < remoteCompressedSize
    = TryIncrementalUpdate localUncompressesSize remoteUncompressedSize hash
  where
    estimatedIncrementalUpdateSize =
      (remoteUncompressedSize - localUncompressesSize)
         `div` estimatedCompressionFactor

    estimatedCompressionFactor
      | disableTransportCompression = 1
      | serverSupportsGzipTransport = 10
      | otherwise                   = 1

-- otherwise
pickUpdateMethod _ _
    IndexManifestResponse {
      indexManifestCompressedInfo = (remoteCompressedSize, hash)
    }
    = TryFullUpdate remoteCompressedSize hash


data LocalIndexFileInfo = NoLocalIndexFile
                        | HaveLocalIndexFile Integer

checkLocalIndex :: RepositoryInfo -> IO ()
checkLocalIndex repo = do
    let localIndexTar = repoLocalIndexFileUncompressed repo
    exists <- doesFileExist localIndexTar
    if exists
      then do size <- withFile localIndexTar ReadMode hFileSize
              return (HaveLocalIndexFile size)
      else return NoLocalIndexFile


data HttpSession m = HttpSession {
       downloadSmallFile :: Int -> URL -> m (Either DownloadError (ByteString, ResponseInfo))
       downloadFile      :: Int -> URL -> FilePath -> m (Either DownloadError ResponseInfo)
     }

HttpSession
  downloadSmallFile -- takes upper limit
  downloadFile -- takes precise size
  missing: downloadByteRange

RepositoryInfo
  repoLocalDir -- local cache, NOT a local repository

DownloadSettings
  missing: some way of bootstrapping security (known root keys, in particular)
  disableTimestampVerification: verification of expiry dates (in case local clocks are wrong, or in case primary server is down for time longer than expiry date on snapshot file)
  disableTransportCompression: because of potential security concerns (e.g. buffer overruns in decompression algorithm, possibly in maliciously created evil compressed files)