Edit: check my answer below for 250 under pure JavaScript.
2852 243 characters using LiveScript (No Regex! Not fully golfed - could be improved)
L=(.0==\\)
A=->it.forEach?&&it.0!=\\
V=(.toFixed?)
S=(a,b,t=-1,l=0)->|L a=>[\\,S(a.1,b,t,l+1)];|A a=>(map (->S(a[it],b,t,l)),[0 1]);|a==l+-1=>S(b,0,l+-1,0)||a|l-1<a=>a+t;|_=>a
R=(a)->|L a=>[\\,R a.1]|(A a)&&(L a.0)=>R(S(R(a.0),R(a.1)).1)|_=>a
Test:
a = [\\,[\\,[1 [1 0]]]]
b = [\\,[\\,[1 [1 [1 0]]]]]
console.log R [a, b]
# outputs ["\\",["\\",[1,[1,[1,[1,[1,[1,[1,[1,[1,0]]]]]]]]]]]
Which is 3^2=9
, as stated on OP.
If anyone is curious, here is an extended version with some comments:
# Just type checking
λ = 100
isλ = (.0==λ)
isA = -> it.forEach? && it.0!=λ
isV = (.toFixed?)
# Performs substitutions in trees
# a: trees to perform substitution in
# b: substitute bound variables by this, if != void
# f: add this value to all unbound variables
# l: internal (depth)
S = (a,b,t=-1,l=0) ->
switch
| isλ a => [λ, (S a.1, b, t, l+1)]
| isA a => [(S a.0, b, t, l), (S a.1, b, t, l)]
| a == l - 1 => (S b, 0, (l - 1), 0) || a
| l - 1 < a < 100 => a + t
| _ => a
# Performs the beta-reduction
R = (a) ->
switch
| (isλ a) => [λ,R a.1]
| (isA a) && (isλ a.0) => R(S(R(a.0),R(a.1)).1)
| _ => a
# Test
a = [λ,[λ,[1 [1 0]]]]
b = [λ,[λ,[1 [1 [1 0]]]]]
console.log show R [a, b]