Navigation Menu

Skip to content

Instantly share code, notes, and snippets.

@jsl
Created September 1, 2016 01:33
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save jsl/13760c9a185e1d0ce24193c2065d4e20 to your computer and use it in GitHub Desktop.
Save jsl/13760c9a185e1d0ce24193c2065d4e20 to your computer and use it in GitHub Desktop.
diffusion change
diff --git a/src/Data/Diffusion/Common.hs b/src/Data/Diffusion/Common.hs
index a86307a..a8d1850 100644
--- a/src/Data/Diffusion/Common.hs
+++ b/src/Data/Diffusion/Common.hs
@@ -18,6 +18,8 @@ import Data.Hashable (Hashable())
import Data.Typeable (Typeable())
import Data.Binary (Binary())
import Control.DeepSeq (NFData)
+import Data.List (find)
+import Data.List (isPrefixOf)
import GHC.IO.Exception (ExitCode (..))
@@ -44,15 +46,26 @@ curlCmd :: String -- ^ The URL to fetch
-> Action ()
curlCmd url destfile = cmd "curl" [url, "-s", "-o", destfile]
+isNewline :: Char -> Bool
+isNewline c = (c == '\r') || (c == '\n')
+
+rstrip :: [Char] -> [Char]
+rstrip = reverse . dropWhile isNewline . reverse
+
-- | Fetches the ETag at the given URL. If no ETag is given by the server,
-- we use the UTC Time that the file was retrieved, so the file will be pretty
-- much downloaded every time.
getEtag :: String -- ^ The URL of the file
-> Action String -- ^ The Etag or UTC Time that this function was called
getEtag url = do
- (Exit c, Stdout out) <- cmd Shell $ "curl -I -L -s " ++ url ++ " | grep ETag"
- case c of
- ExitSuccess -> return out
- ExitFailure _ -> do
+ (Exit c, Stdout out) <- cmd Shell $ "curl -I -L -s " ++ url
+ let etag = find (isPrefixOf "ETag") $ lines out
+
+ case (c, etag) of
+ (ExitSuccess, Just tag) -> return $ rstrip tag
+ (_, _) -> handleFailure
+
+ where
+ handleFailure = do
c' <- liftIO getCurrentTime
return $ show c'
diff --git a/src/GenRegionMap.hs b/src/GenRegionMap.hs
index 4cf0f93..3b93307 100644
--- a/src/GenRegionMap.hs
+++ b/src/GenRegionMap.hs
@@ -13,7 +13,7 @@ opts = shakeOptions { shakeFiles = dataDir
buildMap :: IO ()
buildMap = shakeArgs opts $ do
etagOracle <- addOracle $ \(URL url) -> getEtag url
-
+
want [".diffusion/sub-map.osm.pbf"]
".diffusion/sub-map.osm.pbf" *> \_ -> do
@@ -31,10 +31,6 @@ buildMap = shakeArgs opts $ do
, "file=\".diffusion/sub-map.osm.pbf\""
]
- -- "etagtest" ~> do
- -- r <- getEtag "http://download.geofabrik.de/south-america-latest.osm.pbf"
- -- putNormal $ "hey got: " ++ show r
-
"clean" ~> removeFilesAfter ".diffusion" ["//*"]
".diffusion/full-map.osm.pbf" *> \f -> do
@@ -49,7 +45,7 @@ buildMap = shakeArgs opts $ do
".diffusion/osmosis/bin/osmosis" *> \_ -> do
need [".diffusion/osmosis-latest.tgz"]
cmd "tar" ["xvzf", ".diffusion/osmosis-latest.tgz", "-C", ".diffusion/osmosis"]
-
+
".diffusion/osmosis-latest.tgz" *> \f -> do
ourl <- getReqEnv "OSMOSIS_URL"
curlCmd ourl f
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment