flickrtouchr.py

   1 #!/usr/bin/env python
   2
   3 #
   4 # FlickrTouchr - a simple python script to grab all your photos from flickr,
   5 #                dump into a directory - organised into folders by set -
   6 #                along with any favourites you have saved.
   7 #
   8 #                You can then sync the photos to an iPod touch.
   9 #
  10 # Version:       1.2
  11 #
  12 # Original Author:      colm - AT - allcosts.net  - Colm MacCarthaigh - 2008-01-21
  13 #
  14 # Modified by:                  Dan Benjamin - http://hivelogic.com
  15 #
  16 # License:                      Apache 2.0 - http://www.apache.org/licenses/LICENSE-2.0.html
  17 #
  18
  19 import xml.dom.minidom
  20 import webbrowser
  21 import urlparse
  22 import urllib2
  23 import unicodedata
  24 import cPickle
  25 import md5
  26 import sys
  27 import os
  28
  29 API_KEY       = "e224418b91b4af4e8cdb0564716fa9bd"
  30 SHARED_SECRET = "7cddb9c9716501a0"
  31
  32 #
  33 # Utility functions for dealing with flickr authentication
  34 #
  35 def getText(nodelist):
  36     rc = ""
  37     for node in nodelist:
  38         if node.nodeType == node.TEXT_NODE:
  39             rc = rc + node.data
  40     return rc.encode("utf-8")
  41
  42 #
  43 # Get the frob based on our API_KEY and shared secret
  44 #
  45 def getfrob():
  46     # Create our signing string
  47     string = SHARED_SECRET + "api_key" + API_KEY + "methodflickr.auth.getFrob"
  48     hash   = md5.new(string).digest().encode("hex")
  49
  50     # Formulate the request
  51     url    = "http://api.flickr.com/services/rest/?method=flickr.auth.getFrob"
  52     url   += "&api_key=" + API_KEY + "&api_sig=" + hash
  53
  54     try:
  55         # Make the request and extract the frob
  56         response = urllib2.urlopen(url)
  57
  58         # Parse the XML
  59         dom = xml.dom.minidom.parse(response)
  60
  61         # get the frob
  62         frob = getText(dom.getElementsByTagName("frob")[0].childNodes)
  63
  64         # Free the DOM
  65         dom.unlink()
  66
  67         # Return the frob
  68         return frob
  69
  70     except:
  71         raise "Could not retrieve frob"
  72
  73 #
  74 # Login and get a token
  75 #
  76 def froblogin(frob, perms):
  77     string = SHARED_SECRET + "api_key" + API_KEY + "frob" + frob + "perms" + perms
  78     hash   = md5.new(string).digest().encode("hex")
  79
  80     # Formulate the request
  81     url    = "http://api.flickr.com/services/auth/?"
  82     url   += "api_key=" + API_KEY + "&perms=" + perms
  83     url   += "&frob=" + frob + "&api_sig=" + hash
  84
  85     # Tell the user what's happening
  86     print "In order to allow FlickrTouchr to read your photos and favourites"
  87     print "you need to allow the application. Please press return when you've"
  88     print "granted access at the following url (which should have opened"
  89     print "automatically)."
  90     print
  91     print url
  92     print
  93     print "Waiting for you to press return"
  94
  95     # We now have a login url, open it in a web-browser
  96     webbrowser.open_new(url)
  97
  98     # Wait for input
  99     sys.stdin.readline()
 100
 101     # Now, try and retrieve a token
 102     string = SHARED_SECRET + "api_key" + API_KEY + "frob" + frob + "methodflickr.auth.getToken"
 103     hash   = md5.new(string).digest().encode("hex")
 104
 105     # Formulate the request
 106     url    = "http://api.flickr.com/services/rest/?method=flickr.auth.getToken"
 107     url   += "&api_key=" + API_KEY + "&frob=" + frob
 108     url   += "&api_sig=" + hash
 109
 110     # See if we get a token
 111     try:
 112         # Make the request and extract the frob
 113         response = urllib2.urlopen(url)
 114
 115         # Parse the XML
 116         dom = xml.dom.minidom.parse(response)
 117
 118         # get the token and user-id
 119         token = getText(dom.getElementsByTagName("token")[0].childNodes)
 120         nsid  = dom.getElementsByTagName("user")[0].getAttribute("nsid")
 121
 122         # Free the DOM
 123         dom.unlink()
 124
 125         # Return the token and userid
 126         return (nsid, token)
 127     except:
 128         raise "Login failed"
 129
 130 #
 131 # Sign an arbitrary flickr request with a token
 132 #
 133 def flickrsign(url, token):
 134     query  = urlparse.urlparse(url).query
 135     query += "&api_key=" + API_KEY + "&auth_token=" + token
 136     params = query.split('&')
 137
 138     # Create the string to hash
 139     string = SHARED_SECRET
 140
 141     # Sort the arguments alphabettically
 142     params.sort()
 143     for param in params:
 144         string += param.replace('=', '')
 145     hash   = md5.new(string).digest().encode("hex")
 146
 147     # Now, append the api_key, and the api_sig args
 148     url += "&api_key=" + API_KEY + "&auth_token=" + token + "&api_sig=" + hash
 149
 150     # Return the signed url
 151     return url
 152
 153 #
 154 # Grab the photo from the server
 155 #
 156 def getphoto(imgurl, filename):
 157     # Grab the image file
 158     response = urllib2.urlopen(imgurl)
 159     data = response.read()
 160
 161     # Save the file!
 162     fh = open(filename, "w")
 163     fh.write(data)
 164     fh.close()
 165
 166     return filename
 167
 168 ######## Main Application ##########
 169 if __name__ == '__main__':
 170
 171     # The first, and only argument needs to be a directory
 172     try:
 173         os.chdir(sys.argv[1])
 174     except:
 175         print "usage: %s directory" % sys.argv[0]
 176         sys.exit(1)
 177
 178     # First things first, see if we have a cached user and auth-token
 179     try:
 180         cache = open("touchr.frob.cache", "r")
 181         config = cPickle.load(cache)
 182         cache.close()
 183
 184     # We don't - get a new one
 185     except:
 186         (user, token) = froblogin(getfrob(), "read")
 187         config = { "version":1 , "user":user, "token":token }
 188
 189         # Save it for future use
 190         cache = open("touchr.frob.cache", "w")
 191         cPickle.dump(config, cache)
 192         cache.close()
 193
 194     # Now, construct a query for the list of photo sets
 195     url  = "http://api.flickr.com/services/rest/?method=flickr.photosets.getList"
 196     url += "&user_id=" + config["user"]
 197     url  = flickrsign(url, config["token"])
 198
 199     # get the result
 200     response = urllib2.urlopen(url)
 201
 202     # Parse the XML
 203     dom = xml.dom.minidom.parse(response)
 204
 205     # Get the list of Sets
 206     sets =  dom.getElementsByTagName("photoset")
 207
 208     # For each set - create a url
 209     urls = []
 210     for set in sets:
 211         pid = set.getAttribute("id")
 212         dir = getText(set.getElementsByTagName("title")[0].childNodes)
 213         dir = unicodedata.normalize('NFKD', dir.decode("utf-8", "ignore")).encode('ASCII', 'ignore') # Normalize to ASCII
 214
 215         # Build the list of photos
 216         url   = "http://api.flickr.com/services/rest/?method=flickr.photosets.getPhotos"
 217         url  += "&photoset_id=" + pid
 218
 219         # Append to our list of urls
 220         urls.append( (url , dir) )
 221
 222     # Free the DOM memory
 223     dom.unlink()
 224
 225     # Add the photos which are not in any set
 226     url   = "http://api.flickr.com/services/rest/?method=flickr.photos.getNotInSet"
 227     urls.append( (url, "No Set") )
 228
 229     # Add the user's Favourites
 230     url   = "http://api.flickr.com/services/rest/?method=flickr.favorites.getList"
 231     urls.append( (url, "Favourites") )
 232
 233     # Time to get the photos
 234     inodes = {}
 235     for (url , dir) in urls:
 236         # Create the directory
 237         try:
 238             os.makedirs(dir)
 239         except:
 240             pass
 241
 242         # Get 500 results per page
 243         url += "&per_page=500"
 244         pages = page = 1
 245
 246         # Get Date-Taken and Original-size URL for each result photo
 247         url  += "&extras=date_taken,url_o"
 248
 249         while page <= pages:
 250             request = url + "&page=" + str(page)
 251
 252             # Sign the url
 253             request = flickrsign(request, config["token"])
 254
 255             # Make the request
 256             response = urllib2.urlopen(request)
 257
 258             # Parse the XML
 259             dom = xml.dom.minidom.parse(response)
 260
 261             # Get the total
 262             pages = int(dom.getElementsByTagName("photo")[0].parentNode.getAttribute("pages"))
 263
 264             # Grab the photos
 265             for photo in dom.getElementsByTagName("photo"):
 266                 # Tell the user we're grabbing the file
 267                 print photo.getAttribute("title").encode("utf8") + " ... in set ... " + dir
 268
 269                 # Grab the id
 270                 photoid = photo.getAttribute("id")
 271
 272                 # Grab the taken date
 273                 taken = photo.getAttribute("datetaken")
 274                 taken = taken.replace(":","").replace("-","").replace(" ","")
 275
 276                 # Build the target filename
 277                 target = dir + "/" + taken + "-" + photoid + ".jpg"
 278
 279                 # Skip files that exist
 280                 if os.access(target, os.R_OK):
 281                     inodes[photoid] = target
 282                     continue
 283
 284                 # Look it up in our dictionary of inodes first
 285                 if photoid in inodes and inodes[photoid] and os.access(inodes[photoid], os.R_OK):
 286                     # woo, we have it already, use a hard-link
 287                     os.link(inodes[photoid], target)
 288                 else:
 289                     # Get URL to the "Original" size of the photo
 290                     imgurl = photo.getAttribute("url_o")
 291
 292                     # Grab image and save to local file
 293                     if imgurl:
 294                         inodes[photoid] = getphoto(imgurl, target)
 295                     else:
 296                         print "Failed to retrieve URL for photo id " + photoid
 297
 298             # Move on the next page
 299             page = page + 1