Using Node and CouchDB to get things done
As I mentioned earlier, I love JavaScript. Today, I'd like to talk about how to use it to get (fun) things done.
As I mentioned earlier, I love JavaScript. Today, I'd like to talk about how to use it to get (fun) things done. We are going to make a node data-mining script, and store its work in couch.
Let's make an app that uses node.js and CouchDB to grab location data about the awesome game Glitch, and store it for other apps we will make in later articles.
In node.js, REST is ridiculously easy. We are going to use restler to make it even easier.
Before we begin, make sure you have these installed: node.js and CouchDB. Here are a bunch of binaries. Extract these somewhere, and run "bin/rcouch start" to run a local couch server.
Once complete, and you have couch running, ensure all is well by visiting your futon manager.
It's a common practice to keep node modules needed for a node app in node_modules directory. This is the default location if you use the command-line tool "npm," and is included in the default path when your script runs. Make a directory for your project. On the command-line in the dir, type "npm install restler".
Make a file called getMap.js, and on Mac/Linux chmod +x it, to make it a CLI program. Let's code!
#!/usr/bin/env node var rest = require('restler'); // convenience function for making glitch api requests var glitch = function(url, complete){ rest.get('http://api.glitch.com/simple/' + url) .on('complete', function (data, request){ try{ var out = JSON.parse(data); complete(out); }catch(e){ console.log('response improperly formatted in', url); complete(false); } }) // Glitch always sends status 200, regardless of error, so I need to make sure all is well from the server-error .on('200', function(data, response){ var data = JSON.parse(data||'false'); if (data && data.error){ console.log('error:', data.error); } }); } // get a list of hubs glitch('locations.getHubs', function(data){ if (data){ for (hid in data.hubs){ // get a list of streets for each hub console.log('get hub info (' + hid + '):', data.hubs[hid].name); glitch('locations.getStreets?hub_id=' + hid, function(data){ if (data){ for (sid in data.streets){ // get detailed info for each street console.log('get street info (' + sid + '):', data.streets[sid].name); glitch('locations.streetInfo?street_tsid=' + sid, function(data){ if (data){ console.log('found street', data); } }); } } }); } } });
This will print street info for every street in the game. On Unixes, type "./getMap.js". on Windows, use node getMap.js". You can hit Ctrl-C to break it, once you verify that it works. Some highlights:
- I use a shebang on the first line to make it a standalone CLI program on Unixes.
- I make a convenience function that does error-checking, URL formatting, and JSON parsing.
- I use the anonymous callback pattern, which some people find hard to read (I like it.) To facilitate those anon-function-haters, you can make all the REST API callbacks separate functions, so they are less indented. Totally up to you.
Ok, so now we are getting all the streets. Sweet. Now, lets store them somewhere. We'll use the excellent cradle library, a client for easy CouchDB usage. First, install with "npm install cradle". Now, make the code look like this:
#!/usr/bin/env node var rest = require('restler'); var cradle = require('cradle'); // convenience function for making glitch api requests var glitch = function(url, complete){ rest.get('http://api.glitch.com/simple/' + url) .on('complete', function (data, request){ try{ var out = JSON.parse(data); complete(out); }catch(e){ console.log('response improperly formatted in', url); complete(false); } }) // Glitch always sends status 200, regardless of error .on('200', function(data, response){ var data = JSON.parse(data||'false'); if (data && data.error){ console.log('error:', data.error); } }); } // get all streets, and save them in couch. var getStreets = function(){ // get a list of hubs glitch('locations.getHubs', function(data){ if (data){ for (hid in data.hubs){ // get a list of streets for each hub console.log('get hub info (' + hid + '):', data.hubs[hid].name); glitch('locations.getStreets?hub_id=' + hid, function(data){ if (data){ for (sid in data.streets){ // get detailed info for each street console.log('get street info (' + sid + '):', data.streets[sid].name); glitch('locations.streetInfo?street_tsid=' + sid, function(data){ if (data){ delete data.ok; data.type='street'; db.save(data.tsid, data); } }); } } }); } } }); } var db = new(cradle.Connection)().database('glitchmap'); db.exists(function (err, exists) { if (err){ console.log('something awful happened with the database.', err); }else{ if (!exists){ db.create(getStreets); }else{ getStreets(); } } });
Some highlights:
- I wrapped the street stuff in a function, for DRYness.
- I create a couch database called "glitchmap", if it doesn't already exist.
- Records are keyed by Glitch's street-id
You can see the data you created by visiting your futon manager.
In a future-installment I will demonstrate how to use this data that you collected in a node/couchdb hybrid webapp.