Recast: Why and How


Ben Newman (Meteor)
Facebook New York
10 November 2014 

{ github,
  twitter,
  instagram,
  facebook
}.com/benjamn

Once upon a time at Quora 


var Answer = Component.extend({
  onLoad: function() {
    var params = { uid: this.viewer, aid: this.answer };

    this.$("#upvote").click(this.bind(function() {
      this.serverCall("upvote", params).send();
    }));

    this.$("#downvote").click(this.bind(function() {
      this.serverCall("downvote", params).send();
    }));

    this.$("body").bind("scroll", this.bind(function() {
      // Every time the viewport scrolls, check visibility.
      if (this.$("#container").inViewport()) {
        this.serverCall("viewed_answer", params).send();
      }
    }));
  }
});

Benefits


  • Improved page load times
  • Decreased memory usage
  • Early events are never lost
  • New HTML works immediately
  • Reduced dependency on jQuery
  • Declarative style?

I'm overselling it.


In professional software development, you can almost never justify climbing the tree to pick medium-hanging fruit when there's plenty of lower-hanging fruit left.

gofmt?


  • Easier to grep
  • Easier to review refactored code
  • Easier to read third party code
  • Strong conventions are good

Sure...


Using blunt tools together can make them slightly more powerful.


But try getting the JavaScript community to adopt a tool like jsfmt.

What we really need is a taller ladder!

Meet Recast:


var recast = require("recast");
var ast = recast.parse(source);
transform(ast); // Anything goes.
console.log(recast.print(ast).code);

recast, v.


  1. to give (a metal object) a different form by melting
    it down and reshaping it.
  2. to form, fashion, or arrange again.
  3. to remodel or reconstruct (a literary work, document,
    sentence, etc.).
  4. to supply (a theater or opera work) with a new cast.

recast.parse

var foo = recast.parse("foo;").program.body[0];

assert.deepEqual(foo, {
  "type": "ExpressionStatement",
  "expression": {
    "type": "Identifier",
    "name": "foo",
    "original": { // Identical to foo.original.expression.
      "type": "Identifier",
      "name": "foo",
      "loc": { "start": { "line": 1, "column": 0 },
               "end": { "line": 1, "column": 3 } }
    },
  },
  "original": {
    "type": "ExpressionStatement",
    "expression": { // Identical to foo.expression.original.
      "type": "Identifier",
      "name": "foo",
      "loc": { "start": { "line": 1, "column": 0 },
               "end": { "line": 1, "column": 4 } }
    }
  }
});

recast.visit


  • AST transformation is really hard!
  • Tried to get this API right many times
  • Still a work in progress (you can help!)
  • Pick any style that works for you
    (recast.visit is only a suggestion!)

Example: => function syntax


Input (ES6): 

[3, 1, 10, 28].sort((a, b) => a - b)


Output (ES5):

First, import some utilities and parse the code:


var recast = require("recast");
var n = recast.types.namedTypes;
var b = recast.types.builders;

var ast = recast.parse(
  "[3, 1, 10, 28].sort((a, b) => a - b)"
);

Next, traverse and modify the syntax tree:


recast.visit(ast, {



















});

Next, traverse and modify the syntax tree:


recast.visit(ast, {
  visitArrowFunctionExpression: function(path) {

















  }
});

Next, traverse and modify the syntax tree:


recast.visit(ast, {
  visitArrowFunctionExpression: function(path) {
    var node = path.node;
















  }
});

Next, traverse and modify the syntax tree:


recast.visit(ast, {
  visitArrowFunctionExpression: function(path) {
    var node = path.node;

    if (!n.BlockStatement.check(node.body)) {



    }










  }
});

Next, traverse and modify the syntax tree:


recast.visit(ast, {
  visitArrowFunctionExpression: function(path) {
    var node = path.node;

    if (!n.BlockStatement.check(node.body)) {
      n.Expression.assert(node.body);


    }










  }
});

Next, traverse and modify the syntax tree:


recast.visit(ast, {
  visitArrowFunctionExpression: function(path) {
    var node = path.node;

    if (!n.BlockStatement.check(node.body)) {
      n.Expression.assert(node.body);
      node.body = b.blockStatement([b.returnStatement(node.body)]);

    }










  }
});

Next, traverse and modify the syntax tree:


recast.visit(ast, {
  visitArrowFunctionExpression: function(path) {
    var node = path.node;

    if (!n.BlockStatement.check(node.body)) {
      n.Expression.assert(node.body);
      node.body = b.blockStatement([b.returnStatement(node.body)]);
      node.expression = false;
    }










  }
});

Next, traverse and modify the syntax tree:


recast.visit(ast, {
  visitArrowFunctionExpression: function(path) {
    var node = path.node;

    if (!n.BlockStatement.check(node.body)) {
      n.Expression.assert(node.body);
      node.body = b.blockStatement([b.returnStatement(node.body)]);
      node.expression = false;
    }

    var funExp = b.functionExpression(
      node.id, node.params, node.body,
      node.generator, node.expression
    );





  }
});

Next, traverse and modify the syntax tree:


recast.visit(ast, {
  visitArrowFunctionExpression: function(path) {
    var node = path.node;

    if (!n.BlockStatement.check(node.body)) {
      n.Expression.assert(node.body);
      node.body = b.blockStatement([b.returnStatement(node.body)]);
      node.expression = false;
    }

    var funExp = b.functionExpression(
      node.id, node.params, node.body,
      node.generator, node.expression
    );

    return b.callExpression(
      b.memberExpression(funExp, b.identifier("bind"), false),
      [b.thisExpression()]
    );
  }
});

recast.print





Instead of simply pretty-printing the whole syntax tree, recast.print tries to recyle the original source code wherever possible.

“Non-destructive partial source transformation”

– Ariya Hidayat

Moral hazard:


  • be the laziest, most productive programmer you know
  • be the the change you want to see in the world

Confession:


I gave into temptation for a time.

All told:


1647 files changed
76555 insertions(+)
78260 deletions(-)

1658 classes converted
3582 classes today
find ~/www/html/js/lib | \
  grep "\.js$" | \
  time parallel ~/www/scripts/bin/classify --update

228.03s user 12.25s system 1229% cpu 19.548 total


In case you're curious about the details, I gave a whole talk about this project back in March: slides, video.

Unexpected difficulties


  • Reindentation
  • Parentheses
  • Whitespace between list items
  • Working around and/or fixing Esprima bugs
  • Transformation is just really hard
  • Performance

Lots of additional benefits come with better tools:


  • Minimal sunk costs
  • Painless rebasing
  • Reliable upgrade scripts
  • Easy to improve and/or replace recast.visit
  • Possible to use a different parser
  • On-the-fly compilation
  • Automatic source maps!

Thanks!

External links:


github.com/{
  benjamn/jsconf-2014,
  facebook/jstransform,
  benjamn/recast,
  benjamn/ast-types,
  facebook/regenerator,
  square/esnext
}

code.facebook.com/projects

What about Traceur?