1- ability to add new nodes during the (sometimes) long run – “alternates” to step in for failed nodes.

4- ability to “combine” (local mini-reduce – post map/pre-partition)

2- ability to partition (pre-reduce function that determines reduce grouping)

3- doing the reduction on romote nodes (not all on the master)

%%% Port of -> http://wiki.apache.org/hadoop/WordCount
word_count_test() ->
Words = [the, quick, brown, fox, jumped, over, the, lazy, dog],
Result =
mapreduce(fun(X) -> {X, 1} end,
fun({Word, _}, Dict) ->
case dict:is_key(Word, Dict) of
true -> dict:store(Word,
dict:fetch(Word, Dict) + 1,
Dict);
false -> dict:store(Word, 1, Dict)
end
end,
dict:new(),
Words,
[node()]),
[ ?assertEqual(case Word of the -> 2; _ -> 1 end,
dict:fetch(Word, Result)) || Word <- Words ].

-module(mapreduce).
-export([mapreduce/5]).

-ifdef(TEST).
-include_lib(”eunit/include/eunit.hrl”).
-endif.

mapreduce(F1, F2, Acc, L, Nodes) ->
%% map F1 to the elements of L on nodes from Nodes
Results = do_map(F1, L, Nodes),

%% collect the results, retrying in the event of failures
Vals = collect(F1, Results, Nodes),

%% perform the reduce operation
lists:foldl(F2, Acc, Vals).

%% exit if we ran out of good nodes
do_map(_F1, _L, []) -> exit(no_nodes);
do_map(F1, L, Nodes) ->
Parent = self(),

%% apply F1 to values of L on remote nodes in a rotating fashion
{Results, _Nodes1} =
lists:foldl(
fun(X, {Acc, [Node | Rest]}) ->
erlang:monitor_node(Node, true),
Fun = fun() -> Parent ! {ok, self(), (catch F1(X))} end,
Pid = spawn(Node, Fun),
{[{Pid, X} | Acc], Rest ++ [Node]}
end, {[], Nodes}, L),
Results.

collect(F1, Results, Nodes) ->
collect(F1, Results, Nodes, []).
collect(F1, Results, Nodes, Acc) ->
{Successes, Failures, RemainingNodes}=
lists:foldl(
fun({Pid, X}, {Successes1, Failures1, Nodes1}) ->
Node = node(Pid),
receive
{ok, Pid, Val} ->
{[Val | Successes1], Failures1, Nodes1};

%% we may receive this message because of call to
%% monitor_node()
{nodedown, Node} ->
{Successes1, [X | Failures1], Nodes1 — Node}
end
end, {[], [], Nodes}, Results),

if Failures =/= [] ->
%% retry the failed computations on the remaining nodes
%% and add the results to the current list of successes
Results2 = do_map(F1, Failures, RemainingNodes),
collect(F1, Results2, RemainingNodes, Successes ++ Acc);
true ->
Successes ++ Acc
end.

-ifdef(TEST).

pass_through_test() ->
?assertEqual([one, two, three],
mapreduce(fun(X) -> X end,
fun(X, Acc) -> lists:append(Acc, [X]) end,
[],
[one, two, three],
[node()])).

multiply_by_2_no_reduce_test() ->
?assertEqual([2, 4, 6],
mapreduce(fun(X) -> X * 2 end,
fun(X, Acc) -> lists:append(Acc, [X]) end,
[],
[1, 2, 3],
[node()])).

multiply_by_2_and_reduce_test() ->
?assertEqual([2, 4, 6],
mapreduce(fun(X) -> X * 2 end,
fun(X, Acc) ->
case lists:member(X, Acc) of
false -> lists:append(Acc, [X]);
_ -> Acc
end
end,
[],
[1, 2, 2, 1, 3],
[node()])).

-endif.