Skip to content

Instantly share code, notes, and snippets.

@samgrover
Created April 16, 2019 17:51
Show Gist options
  • Save samgrover/26939e0b68a26b7fe03a7a4bd37bdcf6 to your computer and use it in GitHub Desktop.
Save samgrover/26939e0b68a26b7fe03a7a4bd37bdcf6 to your computer and use it in GitHub Desktop.
Parse the Swarm/Foursquare exported data and create entries in Day One using their command line tool.
#!/usr/bin/env python
# Parse the Swarm/Foursquare exported data and create entries in Day One using their command line tool.
# Day One command line tool available at: http://dayoneapp.com/support/CLI
import sys
import json
import requests
import subprocess
import time
from datetime import timezone, timedelta, datetime
# Foursquare API Keys. Docs at https://developer.foursquare.com/docs
# Needed to get venue coordinates, but only if you set PROCESS_VENUE_COORDS to True.
PROCESS_VENUE_COORDS = False
FOURSQUARE_CLIENT_ID = "YOUR_FOURSQUARE_CLIENT_ID"
FOURSQUARE_CLIENT_SECRET = "YOUR_FOURSQUARE_CLIENT_SECRET"
# Indicates which entries to process for this run.
# Only needed if you're processing venue coords on a limited API plan, or have some other constraint.
START = 0
END = 100
# The name of the journal you want to add the entries into.
JOURNAL_NAME = "YOUR_DAY_ONE_JOURNAL_NAME"
# Entry tags
TAGS = ["Foursquare", "Swarm"]
# A placeholder for entries that have no location/venue name.
NO_NAME_PLACEHOLDER = r"¯\_(ツ)_/¯"
ERROR_MARKER = "<<<<>>>>"
# Set to True to execute the commands to create entries. Otherwise they are just printed out.
EXECUTE_COMMAND = False
def get_JSON(filename):
with open(filename) as json_data:
d = json.load(json_data)
return d
def related_item_url_from_checkin_id(checkin_id):
return f"https://www.swarmapp.com/checkin/{checkin_id}"
def photos_with_checkin_id(incoming_checkin_id):
matching_photos = []
for a_photo_item in photo_items:
related_item_url = a_photo_item["relatedItemUrl"]
checkin_id = related_item_url.split('/')[-1]
if checkin_id == incoming_checkin_id:
a_processed_photo = {
"checkinId": checkin_id,
"fullUrl": a_photo_item["fullUrl"],
"width": a_photo_item["width"],
"height": a_photo_item["height"]
}
matching_photos.append(a_processed_photo)
return matching_photos
def get_lat_lng(location):
return (location["lat"], location["lng"])
def get_venue_location(venue_id):
# Uncomment the following line if you want to rate limit the call for venue info
# time.sleep(2)
coords = (0, 0)
if PROCESS_VENUE_COORDS is False:
return coords
url = f'https://api.foursquare.com/v2/venues/{venue_id}'
params = dict(
v='20190323',
client_id=FOURSQUARE_CLIENT_ID,
client_secret=FOURSQUARE_CLIENT_SECRET,
)
resp = requests.get(url=url, params=params)
if resp.status_code == 200:
data = json.loads(resp.text)
# print(data)
meta = data["meta"]
if meta["code"] == 200:
coords = get_lat_lng(data["response"]["venue"]["location"])
else:
print(ERROR_MARKER)
print(f"Error retrieving details for venue: {venue_id}")
print(json.dumps(meta, indent=4))
print(ERROR_MARKER)
return coords
def text_for_name(name):
text = f"I'm at {name}"
return text
def add_shout(item, text):
shout = ""
if "shout" in item:
shout = item["shout"]
text += f"\n{shout}"
return text
return text
if len(sys.argv) != 3:
print("This script requires the following arguments:")
print("swarm-day-one-import.py <checkins.json> <photos.json>")
exit(0)
checkins = get_JSON(sys.argv[1])
checkin_items = checkins['items']
TOTAL = checkins['count']
photos = get_JSON(sys.argv[2])
photo_items = photos['items']
# Pre process all the photos and add them to the checkins
for a_checkin_item in checkin_items:
if a_checkin_item["type"] == "checkin":
checkin_id = a_checkin_item["id"]
matching_photos = photos_with_checkin_id(checkin_id)
if len(matching_photos) > 0:
a_checkin_item["photos"] = matching_photos
for item in checkin_items[START:END]:
text = ''
path_to_photos = []
lat = 0
lng = 0
if item["type"] == "checkin":
if "venue" in item:
venue = item["venue"]
name = venue["name"]
text = text_for_name(name)
text = add_shout(item, text)
photos = []
if "photos" in item:
photos = item["photos"]
for a_photo in photos:
photo_url = a_photo["fullUrl"]
filename = photo_url.split('/')[-1]
path_to_filename = f"images/{filename}"
path_to_photos.append(path_to_filename)
print(path_to_filename)
(lat, lng) = get_venue_location(venue["id"])
else:
# These don't have venue or location.
text = text_for_name(NO_NAME_PLACEHOLDER)
text = add_shout(item, text)
elif item["type"] == "venueless":
# Process these ones that have location instead of venue. type = venueless.
# None of these have photos in my data set so we will just ignore processing that in here.
location = item["location"]
name = location["name"]
text = text_for_name(name)
text = add_shout(item, text)
(lat, lng) = get_lat_lng(location)
elif item["type"] == "shout":
location = item["location"]
text = text_for_name(NO_NAME_PLACEHOLDER)
text = add_shout(item, text)
timestamp = datetime.fromtimestamp(item["createdAt"], timezone.utc)
timestr = '--isoDate=' + timestamp.strftime("%Y-%m-%dT%H:%M:%SZ")
dayone_command = ['dayone2', 'new', text, '--journal', JOURNAL_NAME, timestr]
tz = timezone(timedelta(minutes=item["timeZoneOffset"]))
dayone_command.extend(['-z', str(tz)])
if len(TAGS) > 0:
dayone_command.extend(['-t'] + TAGS)
if len(path_to_photos) > 0:
dayone_command.extend(['--photos'] + path_to_photos)
if lat != 0 and lng != 0:
dayone_command.extend(['--coordinate', str(lat), str(lng)])
print(" ".join(dayone_command))
if EXECUTE_COMMAND is True:
subprocess.call(dayone_command)
print(f"Start = {START}")
print(f"End = {END}")
print(f"Total = {TOTAL}")
print(f"Remaining = {TOTAL - END}")
@samgrover
Copy link
Author

And thanks to @otaviocc for updating it so it remains useful.

@otaviocc
Copy link

@samgrover my pleasure!

@Kurtois
Copy link

Kurtois commented Dec 13, 2024

@samgrover - Sent an email to your samgrover at Gmail address with no response on 11.22.24.

I see in this thread you no longer use Swarm. Are you using something as a replacement?

@otaviocc - Have you recently used (within past 3 months, for example) your updated script recorded here with Swarm?

Also, same question I posed to @samgrover. Are you using something other than Swarm in 2024?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment