Created
April 12, 2020 19:48
-
-
Save cwebbbi/88af2d8119142c873a9ec98febd44f87 to your computer and use it in GitHub Desktop.
A Power Query M query for use with Power BI incremental refresh that gets a list of top stories from the BBC News RSS feed
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
let | |
//Connect to the BBC News Top Stories RSS feed | |
//and create a nicely formatted table | |
Source = Xml.Tables(Web.Contents("http://feeds.bbci.co.uk/news/rss.xml")), | |
#"Changed Type" = Table.TransformColumnTypes(Source,{{"Attribute:version", Int64.Type}}), | |
channel = #"Changed Type"{0}[channel], | |
#"Changed Type1" = Table.TransformColumnTypes(channel,{{"title", type text}, {"description", type text}, {"link", type text}, {"generator", type text}, {"lastBuildDate", type datetime}, {"copyright", type text}, {"language", type text}, {"ttl", Int64.Type}}), | |
item = #"Changed Type1"{0}[item], | |
#"Changed Type2" = Table.TransformColumnTypes(item,{{"title", type text}, {"description", type text}, {"link", type text}, {"pubDate", type datetime}}), | |
#"Removed Columns" = Table.RemoveColumns(#"Changed Type2",{"guid"}), | |
#"Renamed Columns" = Table.RenameColumns(#"Removed Columns",{{"title", "Title"}, {"description", "Description"}, {"link", "Link"}, {"pubDate", "Publication Date"}}), | |
//Find the current date and time when this query runs | |
CurrentDateTime = DateTimeZone.FixedUtcNow(), | |
//Find yesterday's date | |
PreviousDay = Date.AddDays(DateTime.Date(CurrentDateTime),-1), | |
//Put the current date and time in a new column in the table | |
#"Added Custom" = Table.AddColumn(#"Renamed Columns", "UTC Data Load Date", each CurrentDateTime), | |
#"Changed Type3" = Table.TransformColumnTypes(#"Added Custom",{{"UTC Data Load Date", type datetimezone}}), | |
//Add the filter required for incremental refresh | |
//Only return rows in this table if: | |
//a) The RangeStart parameter equals yesterday's date, and | |
//b) RangeEnd is not null (which should never be true) | |
#"Filtered Rows" = Table.SelectRows(#"Changed Type3", each DateTime.Date(RangeStart)=PreviousDay and RangeEnd<>null) | |
in | |
#"Filtered Rows" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment