Master takeover protocol.

Implement the takeover protocol that guarantees correct after master
takeover.

5 files changed, 490 insertions, 86 deletions

diff --git a/same/src/main/java/com/orbekk/same/Client.java b/same/src/main/java/com/orbekk/same/Client.java
index c0bf708..cafa9d8 100644
--- a/same/src/main/java/com/orbekk/same/Client.java
+++ b/same/src/main/java/com/orbekk/same/Client.java
@@ -174,17 +174,15 @@ public class Client {
                     request.getMasterId() <= masterInfo.getMasterId()) {
                 logger.warn("{} tried to take over, but current master is " +
                         "{}. Ignoring", masterInfo); 
+                done.run(MasterTakeoverResponse.newBuilder().setSuccess(false).build());
                 return;
             }
             abortMasterElection();
-            long highestRevision = 0;
-            if (masterInfo != null && request.getNetworkName().equals(masterInfo.getNetworkName())) {
-                highestRevision = revision.get();
-            }
             masterInfo = request;
-            setConnectionState(ConnectionState.STABLE);
+            setConnectionState(ConnectionState.UNSTABLE);
             done.run(MasterTakeoverResponse.newBuilder()
-                    .setHighestKnownRevision(highestRevision)
+                    .setSuccess(true)
+                    .setClientState(getClientState())
                     .build());
         }
 
@@ -210,6 +208,21 @@ public class Client {
                     ServicesPbConversion.componentsToPb(state.getComponents()));
             done.run(response.build());
         }
+
+        @Override
+        public void masterTakeoverFinished(RpcController controller,
+                MasterState request, RpcCallback<Empty> done) {
+            if (masterInfo != null && request.getMasterId() < masterInfo.getMasterId()) {
+                logger.warn("MasterTakeoverFinished({}) failed. Current master: {}", request, masterInfo);
+            }
+            if (revision.get() < masterInfo.getRevision()) {
+                logger.error("My revision ({}) < master revision ({}). Possible inconsistency.",
+                        revision.get(), masterInfo.getRevision());
+            }
+            masterInfo = request;
+            setConnectionState(ConnectionState.STABLE);
+            done.run(Empty.getDefaultInstance());
+        }
     };
     
     private class MasterStarter implements Runnable {
diff --git a/same/src/main/java/com/orbekk/same/Master.java b/same/src/main/java/com/orbekk/same/Master.java
index 678baf2..0571896 100644
--- a/same/src/main/java/com/orbekk/same/Master.java
+++ b/same/src/main/java/com/orbekk/same/Master.java
@@ -15,7 +15,10 @@
  */
 package com.orbekk.same;
 
+import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.slf4j.Logger;
@@ -26,8 +29,11 @@ import com.google.protobuf.RpcController;
 import com.orbekk.protobuf.Rpc;
 import com.orbekk.same.Services.ClientState;
 import com.orbekk.same.Services.Empty;
+import com.orbekk.same.Services.FullStateResponse;
+import com.orbekk.same.Services.MasterState;
 import com.orbekk.same.Services.MasterTakeoverResponse;
 import com.orbekk.same.State.Component;
+import com.orbekk.util.RpcList;
 
 public class Master {
     private Logger logger = LoggerFactory.getLogger(getClass());
@@ -57,6 +63,163 @@ public class Master {
         }
     }
     
+    private class MasterTakeover implements Runnable {
+        final List<Services.ClientState> clientStates = new ArrayList<Services.ClientState>();
+        final AtomicBoolean aborted = new AtomicBoolean(false);
+        final List<String> clients = new CopyOnWriteArrayList<String>();
+        final MasterState newMaster;
+        
+        private class TakeoverCallback implements RpcCallback<MasterTakeoverResponse> {
+            final String client;
+
+            public TakeoverCallback(String client) {
+                this.client = client;
+            }
+            
+            @Override public void run(MasterTakeoverResponse response) {
+                if (response == null) {
+                    clients.remove(client);
+                } else if (!response.getSuccess()) {
+                    aborted.set(true);
+                } else {
+                    clientStates.add(response.getClientState());
+                }
+            }
+        }
+        
+        private class FullStateCallback implements RpcCallback<FullStateResponse> {
+            @Override public void run(FullStateResponse response) {
+                if (response != null) {
+                    for (Services.Component componentPb : response.getComponentList()) {
+                        state.update(componentPb.getId(), componentPb.getData(),
+                                componentPb.getRevision());
+                        updateRevision(componentPb.getRevision());
+                    }
+                }
+            }
+        }
+        
+        private class RemoveClientCallback<T> implements RpcCallback<T> {
+            final String client;
+            
+            public RemoveClientCallback(String client) {
+                this.client = client;
+            }
+            
+            @Override public void run(T response) {
+                if (response == null) {
+                    clients.remove(client);
+                }
+            }
+        }
+        
+        public MasterTakeover(List<String> clients, MasterState newMaster) {
+            this.clients.addAll(clients);
+            this.newMaster = newMaster;
+        }
+
+        private void sendTakeovers() throws InterruptedException {
+            RpcList rpcs = new RpcList();
+            for (String location : clients) {
+                Services.Client client = connections.getClient0(location);
+                if (client == null) {
+                    clients.remove(location);
+                } else {
+                    Rpc rpc = rpcf.create();
+                    client.masterTakeover(rpc, newMaster, new TakeoverCallback(location));
+                    rpcs.add(rpc);
+                }
+            }
+            rpcs.awaitAll();
+        }
+        
+        private ClientState getBestClient(List<ClientState> clients) {
+            if (clients.isEmpty()) {
+                return null;
+            }
+            ClientState best = clients.get(0);
+            for (ClientState client : clients) {
+                if (client.getRevision() > best.getRevision()) {
+                    best = client;
+                }
+            }
+            return best;
+        }
+        
+        private void getMostRecentState() throws InterruptedException {
+            boolean successful = false;
+            while (!successful && !aborted.get()) {
+                ClientState bestClient = getBestClient(clientStates);
+                if (bestClient == null) {
+                    successful = true;
+                    continue;
+                }
+                Services.Client client = connections.getClient0(bestClient.getLocation());
+                if (client == null) {
+                    clients.remove(bestClient.getLocation());
+                    continue;
+                }
+                
+                Rpc rpc = rpcf.create();
+                FullStateCallback done = new FullStateCallback();
+                client.getFullState(rpc, Empty.getDefaultInstance(), done);
+                rpc.await();
+                successful = rpc.isOk();
+                
+                if (!successful) {
+                    clients.remove(bestClient.getLocation());
+                }
+            }
+        }
+        
+        private void sendFullState() throws InterruptedException {
+            RpcList rpcs = new RpcList();
+            for (String location : clients) {
+                Services.Client client = connections.getClient0(location);
+                if (client == null) {
+                    clients.remove(client);
+                    continue;
+                }
+                RemoveClientCallback<Empty> done = new RemoveClientCallback<Empty>(location);
+                for (Component component : state.getComponents()) {
+                    Services.Component componentPb = ServicesPbConversion.componentToPb(component);
+                    Rpc rpc = rpcf.create();
+                    client.setState(rpc, componentPb, done);
+                    rpcs.add(rpc);
+                }
+            }
+            rpcs.awaitAll();
+        }
+        
+        private void finishTakeover() throws InterruptedException {
+            RpcList rpcs = new RpcList();
+            for (String location : clients) {
+                Services.Client client = connections.getClient0(location);
+                if (client == null) {
+                    clients.remove(client);
+                    continue;
+                }
+                RemoveClientCallback<Empty> done = new RemoveClientCallback<Empty>(location);
+                Rpc rpc = rpcf.create();
+                client.masterTakeoverFinished(rpc, newMaster, done);
+                rpcs.add(rpc);
+            }
+            rpcs.awaitAll();
+        }
+        
+        @Override public void run() {
+            try {
+                sendTakeovers();
+                getMostRecentState();
+                sendFullState();
+                finishTakeover();
+            } catch (InterruptedException e) {
+                // Abort master takeover.
+                aborted.set(true);
+            }
+        }
+    }
+    
     public static Master create(ConnectionManager connections,
             String myUrl, String networkName,
             String myLocation, RpcFactory rpcf) {
@@ -94,7 +257,6 @@ public class Master {
         @Override public void joinNetworkRequest(RpcController controller,
                 ClientState request, RpcCallback<Empty> done) {
             sendInitialMasterTakeover(request.getLocation());
-            sendFullState(request.getLocation());
             addParticipant(request.getLocation());
             done.run(Empty.getDefaultInstance());
         }
@@ -158,13 +320,24 @@ public class Master {
         sendComponents(clientLocation, components);
     }
     
-    private void sendInitialMasterTakeover(String clientLocation) {
+    private synchronized void sendInitialMasterTakeover(String clientLocation) {
         Services.Client client = connections.getClient0(clientLocation);
-        Rpc rpc = rpcf.create();
-        RpcCallback<MasterTakeoverResponse> done =
+        
+        // Step 1: Send takeover.
+        Rpc rpc1 = rpcf.create();
+        RpcCallback<MasterTakeoverResponse> done1 =
                 new RemoveParticipantIfFailsCallback<MasterTakeoverResponse>(
-                        clientLocation, rpc);
-        client.masterTakeover(rpc, getMasterInfo(), done);
+                        clientLocation, rpc1);
+        client.masterTakeover(rpc1, getMasterInfo(), done1);
+        
+        // Step 2: Send all state.
+        sendFullState(clientLocation);
+        
+        // Step 3: Finish takeover.
+        Rpc rpc2 = rpcf.create();
+        RpcCallback<Empty> done2 = new RemoveParticipantIfFailsCallback<Empty>(
+                clientLocation, rpc2);
+        client.masterTakeoverFinished(rpc2, getMasterInfo(), done2);
     }
     
     void performWork() {
@@ -204,18 +377,22 @@ public class Master {
     
     /** This master should take over from an earlier master. */
     public void resumeFrom(State lastKnownState, final int masterId) {
+        for (Component c : lastKnownState.getComponents()) {
+            updateRevision(c.getRevision());
+        }
         state = lastKnownState;
         this.masterId = masterId;
-        
-        for (final String location : state.getList(State.PARTICIPANTS)) {
-            Services.Client client = connections.getClient0(location);
-            final Rpc rpc = rpcf.create();
-            RpcCallback<MasterTakeoverResponse> done = new RemoveParticipantIfFailsCallback<Services.MasterTakeoverResponse>(location, rpc);
-            if (client == null) {
-                removeParticipant(location);
-                continue;
-            }
-            client.masterTakeover(rpc, getMasterInfo(), done);
+        MasterTakeover takeover = new MasterTakeover(
+                state.getList(State.PARTICIPANTS), getMasterInfo());
+        takeover.run();
+    }
+    
+    public void updateRevision(long newRevision) {
+        boolean updated = false;
+        while (!updated) {
+            long expected = revision.get();
+            long update = Math.max(expected, newRevision);
+            updated = revision.compareAndSet(expected, update);
         }
     }
 }
diff --git a/same/src/main/java/com/orbekk/same/SameController.java b/same/src/main/java/com/orbekk/same/SameController.java
index 5eebcdd..8f225c6 100644
--- a/same/src/main/java/com/orbekk/same/SameController.java
+++ b/same/src/main/java/com/orbekk/same/SameController.java
@@ -123,7 +123,7 @@ public class SameController {
     }
     
     public static SameController create(Configuration configuration) {
-//        enableRpcLogging();
+        enableRpcLogging();
         int pport = configuration.getInt("pport");
         String myLocation = configuration.get("localIp") + ":" + pport;
         
diff --git a/same/src/main/java/com/orbekk/same/Services.java b/same/src/main/java/com/orbekk/same/Services.java
index 08c7c4a..266b30e 100644
--- a/same/src/main/java/com/orbekk/same/Services.java
+++ b/same/src/main/java/com/orbekk/same/Services.java
@@ -3473,9 +3473,14 @@ public final class Services {
   public interface MasterTakeoverResponseOrBuilder
       extends com.google.protobuf.MessageOrBuilder {
     
-    // optional int64 highest_known_revision = 1;
-    boolean hasHighestKnownRevision();
-    long getHighestKnownRevision();
+    // optional bool success = 2;
+    boolean hasSuccess();
+    boolean getSuccess();
+    
+    // optional .com.orbekk.same.ClientState client_state = 3;
+    boolean hasClientState();
+    com.orbekk.same.Services.ClientState getClientState();
+    com.orbekk.same.Services.ClientStateOrBuilder getClientStateOrBuilder();
   }
   public static final class MasterTakeoverResponse extends
       com.google.protobuf.GeneratedMessage
@@ -3506,18 +3511,32 @@ public final class Services {
     }
     
     private int bitField0_;
-    // optional int64 highest_known_revision = 1;
-    public static final int HIGHEST_KNOWN_REVISION_FIELD_NUMBER = 1;
-    private long highestKnownRevision_;
-    public boolean hasHighestKnownRevision() {
+    // optional bool success = 2;
+    public static final int SUCCESS_FIELD_NUMBER = 2;
+    private boolean success_;
+    public boolean hasSuccess() {
       return ((bitField0_ & 0x00000001) == 0x00000001);
     }
-    public long getHighestKnownRevision() {
-      return highestKnownRevision_;
+    public boolean getSuccess() {
+      return success_;
+    }
+    
+    // optional .com.orbekk.same.ClientState client_state = 3;
+    public static final int CLIENT_STATE_FIELD_NUMBER = 3;
+    private com.orbekk.same.Services.ClientState clientState_;
+    public boolean hasClientState() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public com.orbekk.same.Services.ClientState getClientState() {
+      return clientState_;
+    }
+    public com.orbekk.same.Services.ClientStateOrBuilder getClientStateOrBuilder() {
+      return clientState_;
     }
     
     private void initFields() {
-      highestKnownRevision_ = 0L;
+      success_ = false;
+      clientState_ = com.orbekk.same.Services.ClientState.getDefaultInstance();
     }
     private byte memoizedIsInitialized = -1;
     public final boolean isInitialized() {
@@ -3532,7 +3551,10 @@ public final class Services {
                         throws java.io.IOException {
       getSerializedSize();
       if (((bitField0_ & 0x00000001) == 0x00000001)) {
-        output.writeInt64(1, highestKnownRevision_);
+        output.writeBool(2, success_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeMessage(3, clientState_);
       }
       getUnknownFields().writeTo(output);
     }
@@ -3545,7 +3567,11 @@ public final class Services {
       size = 0;
       if (((bitField0_ & 0x00000001) == 0x00000001)) {
         size += com.google.protobuf.CodedOutputStream
-          .computeInt64Size(1, highestKnownRevision_);
+          .computeBoolSize(2, success_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(3, clientState_);
       }
       size += getUnknownFields().getSerializedSize();
       memoizedSerializedSize = size;
@@ -3663,6 +3689,7 @@ public final class Services {
       }
       private void maybeForceBuilderInitialization() {
         if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+          getClientStateFieldBuilder();
         }
       }
       private static Builder create() {
@@ -3671,8 +3698,14 @@ public final class Services {
       
       public Builder clear() {
         super.clear();
-        highestKnownRevision_ = 0L;
+        success_ = false;
         bitField0_ = (bitField0_ & ~0x00000001);
+        if (clientStateBuilder_ == null) {
+          clientState_ = com.orbekk.same.Services.ClientState.getDefaultInstance();
+        } else {
+          clientStateBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000002);
         return this;
       }
       
@@ -3714,7 +3747,15 @@ public final class Services {
         if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
           to_bitField0_ |= 0x00000001;
         }
-        result.highestKnownRevision_ = highestKnownRevision_;
+        result.success_ = success_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        if (clientStateBuilder_ == null) {
+          result.clientState_ = clientState_;
+        } else {
+          result.clientState_ = clientStateBuilder_.build();
+        }
         result.bitField0_ = to_bitField0_;
         onBuilt();
         return result;
@@ -3731,8 +3772,11 @@ public final class Services {
       
       public Builder mergeFrom(com.orbekk.same.Services.MasterTakeoverResponse other) {
         if (other == com.orbekk.same.Services.MasterTakeoverResponse.getDefaultInstance()) return this;
-        if (other.hasHighestKnownRevision()) {
-          setHighestKnownRevision(other.getHighestKnownRevision());
+        if (other.hasSuccess()) {
+          setSuccess(other.getSuccess());
+        }
+        if (other.hasClientState()) {
+          mergeClientState(other.getClientState());
         }
         this.mergeUnknownFields(other.getUnknownFields());
         return this;
@@ -3765,9 +3809,18 @@ public final class Services {
               }
               break;
             }
-            case 8: {
+            case 16: {
               bitField0_ |= 0x00000001;
-              highestKnownRevision_ = input.readInt64();
+              success_ = input.readBool();
+              break;
+            }
+            case 26: {
+              com.orbekk.same.Services.ClientState.Builder subBuilder = com.orbekk.same.Services.ClientState.newBuilder();
+              if (hasClientState()) {
+                subBuilder.mergeFrom(getClientState());
+              }
+              input.readMessage(subBuilder, extensionRegistry);
+              setClientState(subBuilder.buildPartial());
               break;
             }
           }
@@ -3776,27 +3829,117 @@ public final class Services {
       
       private int bitField0_;
       
-      // optional int64 highest_known_revision = 1;
-      private long highestKnownRevision_ ;
-      public boolean hasHighestKnownRevision() {
+      // optional bool success = 2;
+      private boolean success_ ;
+      public boolean hasSuccess() {
         return ((bitField0_ & 0x00000001) == 0x00000001);
       }
-      public long getHighestKnownRevision() {
-        return highestKnownRevision_;
+      public boolean getSuccess() {
+        return success_;
       }
-      public Builder setHighestKnownRevision(long value) {
+      public Builder setSuccess(boolean value) {
         bitField0_ |= 0x00000001;
-        highestKnownRevision_ = value;
+        success_ = value;
         onChanged();
         return this;
       }
-      public Builder clearHighestKnownRevision() {
+      public Builder clearSuccess() {
         bitField0_ = (bitField0_ & ~0x00000001);
-        highestKnownRevision_ = 0L;
+        success_ = false;
         onChanged();
         return this;
       }
       
+      // optional .com.orbekk.same.ClientState client_state = 3;
+      private com.orbekk.same.Services.ClientState clientState_ = com.orbekk.same.Services.ClientState.getDefaultInstance();
+      private com.google.protobuf.SingleFieldBuilder<
+          com.orbekk.same.Services.ClientState, com.orbekk.same.Services.ClientState.Builder, com.orbekk.same.Services.ClientStateOrBuilder> clientStateBuilder_;
+      public boolean hasClientState() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public com.orbekk.same.Services.ClientState getClientState() {
+        if (clientStateBuilder_ == null) {
+          return clientState_;
+        } else {
+          return clientStateBuilder_.getMessage();
+        }
+      }
+      public Builder setClientState(com.orbekk.same.Services.ClientState value) {
+        if (clientStateBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          clientState_ = value;
+          onChanged();
+        } else {
+          clientStateBuilder_.setMessage(value);
+        }
+        bitField0_ |= 0x00000002;
+        return this;
+      }
+      public Builder setClientState(
+          com.orbekk.same.Services.ClientState.Builder builderForValue) {
+        if (clientStateBuilder_ == null) {
+          clientState_ = builderForValue.build();
+          onChanged();
+        } else {
+          clientStateBuilder_.setMessage(builderForValue.build());
+        }
+        bitField0_ |= 0x00000002;
+        return this;
+      }
+      public Builder mergeClientState(com.orbekk.same.Services.ClientState value) {
+        if (clientStateBuilder_ == null) {
+          if (((bitField0_ & 0x00000002) == 0x00000002) &&
+              clientState_ != com.orbekk.same.Services.ClientState.getDefaultInstance()) {
+            clientState_ =
+              com.orbekk.same.Services.ClientState.newBuilder(clientState_).mergeFrom(value).buildPartial();
+          } else {
+            clientState_ = value;
+          }
+          onChanged();
+        } else {
+          clientStateBuilder_.mergeFrom(value);
+        }
+        bitField0_ |= 0x00000002;
+        return this;
+      }
+      public Builder clearClientState() {
+        if (clientStateBuilder_ == null) {
+          clientState_ = com.orbekk.same.Services.ClientState.getDefaultInstance();
+          onChanged();
+        } else {
+          clientStateBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000002);
+        return this;
+      }
+      public com.orbekk.same.Services.ClientState.Builder getClientStateBuilder() {
+        bitField0_ |= 0x00000002;
+        onChanged();
+        return getClientStateFieldBuilder().getBuilder();
+      }
+      public com.orbekk.same.Services.ClientStateOrBuilder getClientStateOrBuilder() {
+        if (clientStateBuilder_ != null) {
+          return clientStateBuilder_.getMessageOrBuilder();
+        } else {
+          return clientState_;
+        }
+      }
+      private com.google.protobuf.SingleFieldBuilder<
+          com.orbekk.same.Services.ClientState, com.orbekk.same.Services.ClientState.Builder, com.orbekk.same.Services.ClientStateOrBuilder> 
+          getClientStateFieldBuilder() {
+        if (clientStateBuilder_ == null) {
+          clientStateBuilder_ = new com.google.protobuf.SingleFieldBuilder<
+              com.orbekk.same.Services.ClientState, com.orbekk.same.Services.ClientState.Builder, com.orbekk.same.Services.ClientStateOrBuilder>(
+                  clientState_,
+                  getParentForChildren(),
+                  isClean());
+          clientState_ = null;
+        }
+        return clientStateBuilder_;
+      }
+      
       // @@protoc_insertion_point(builder_scope:com.orbekk.same.MasterTakeoverResponse)
     }
     
@@ -6352,6 +6495,11 @@ public final class Services {
           com.orbekk.same.Services.Empty request,
           com.google.protobuf.RpcCallback<com.orbekk.same.Services.FullStateResponse> done);
       
+      public abstract void masterTakeoverFinished(
+          com.google.protobuf.RpcController controller,
+          com.orbekk.same.Services.MasterState request,
+          com.google.protobuf.RpcCallback<com.orbekk.same.Services.Empty> done);
+      
     }
     
     public static com.google.protobuf.Service newReflectiveService(
@@ -6389,6 +6537,14 @@ public final class Services {
           impl.getFullState(controller, request, done);
         }
         
+        @java.lang.Override
+        public  void masterTakeoverFinished(
+            com.google.protobuf.RpcController controller,
+            com.orbekk.same.Services.MasterState request,
+            com.google.protobuf.RpcCallback<com.orbekk.same.Services.Empty> done) {
+          impl.masterTakeoverFinished(controller, request, done);
+        }
+        
       };
     }
     
@@ -6419,6 +6575,8 @@ public final class Services {
               return impl.masterTakeover(controller, (com.orbekk.same.Services.MasterState)request);
             case 3:
               return impl.getFullState(controller, (com.orbekk.same.Services.Empty)request);
+            case 4:
+              return impl.masterTakeoverFinished(controller, (com.orbekk.same.Services.MasterState)request);
             default:
               throw new java.lang.AssertionError("Can't get here.");
           }
@@ -6441,6 +6599,8 @@ public final class Services {
               return com.orbekk.same.Services.MasterState.getDefaultInstance();
             case 3:
               return com.orbekk.same.Services.Empty.getDefaultInstance();
+            case 4:
+              return com.orbekk.same.Services.MasterState.getDefaultInstance();
             default:
               throw new java.lang.AssertionError("Can't get here.");
           }
@@ -6463,6 +6623,8 @@ public final class Services {
               return com.orbekk.same.Services.MasterTakeoverResponse.getDefaultInstance();
             case 3:
               return com.orbekk.same.Services.FullStateResponse.getDefaultInstance();
+            case 4:
+              return com.orbekk.same.Services.Empty.getDefaultInstance();
             default:
               throw new java.lang.AssertionError("Can't get here.");
           }
@@ -6491,6 +6653,11 @@ public final class Services {
         com.orbekk.same.Services.Empty request,
         com.google.protobuf.RpcCallback<com.orbekk.same.Services.FullStateResponse> done);
     
+    public abstract void masterTakeoverFinished(
+        com.google.protobuf.RpcController controller,
+        com.orbekk.same.Services.MasterState request,
+        com.google.protobuf.RpcCallback<com.orbekk.same.Services.Empty> done);
+    
     public static final
         com.google.protobuf.Descriptors.ServiceDescriptor
         getDescriptor() {
@@ -6533,6 +6700,11 @@ public final class Services {
             com.google.protobuf.RpcUtil.<com.orbekk.same.Services.FullStateResponse>specializeCallback(
               done));
           return;
+        case 4:
+          this.masterTakeoverFinished(controller, (com.orbekk.same.Services.MasterState)request,
+            com.google.protobuf.RpcUtil.<com.orbekk.same.Services.Empty>specializeCallback(
+              done));
+          return;
         default:
           throw new java.lang.AssertionError("Can't get here.");
       }
@@ -6555,6 +6727,8 @@ public final class Services {
           return com.orbekk.same.Services.MasterState.getDefaultInstance();
         case 3:
           return com.orbekk.same.Services.Empty.getDefaultInstance();
+        case 4:
+          return com.orbekk.same.Services.MasterState.getDefaultInstance();
         default:
           throw new java.lang.AssertionError("Can't get here.");
       }
@@ -6577,6 +6751,8 @@ public final class Services {
           return com.orbekk.same.Services.MasterTakeoverResponse.getDefaultInstance();
         case 3:
           return com.orbekk.same.Services.FullStateResponse.getDefaultInstance();
+        case 4:
+          return com.orbekk.same.Services.Empty.getDefaultInstance();
         default:
           throw new java.lang.AssertionError("Can't get here.");
       }
@@ -6657,6 +6833,21 @@ public final class Services {
             com.orbekk.same.Services.FullStateResponse.class,
             com.orbekk.same.Services.FullStateResponse.getDefaultInstance()));
       }
+      
+      public  void masterTakeoverFinished(
+          com.google.protobuf.RpcController controller,
+          com.orbekk.same.Services.MasterState request,
+          com.google.protobuf.RpcCallback<com.orbekk.same.Services.Empty> done) {
+        channel.callMethod(
+          getDescriptor().getMethods().get(4),
+          controller,
+          request,
+          com.orbekk.same.Services.Empty.getDefaultInstance(),
+          com.google.protobuf.RpcUtil.generalizeCallback(
+            done,
+            com.orbekk.same.Services.Empty.class,
+            com.orbekk.same.Services.Empty.getDefaultInstance()));
+      }
     }
     
     public static BlockingInterface newBlockingStub(
@@ -6684,6 +6875,11 @@ public final class Services {
           com.google.protobuf.RpcController controller,
           com.orbekk.same.Services.Empty request)
           throws com.google.protobuf.ServiceException;
+      
+      public com.orbekk.same.Services.Empty masterTakeoverFinished(
+          com.google.protobuf.RpcController controller,
+          com.orbekk.same.Services.MasterState request)
+          throws com.google.protobuf.ServiceException;
     }
     
     private static final class BlockingStub implements BlockingInterface {
@@ -6740,6 +6936,18 @@ public final class Services {
           com.orbekk.same.Services.FullStateResponse.getDefaultInstance());
       }
       
+      
+      public com.orbekk.same.Services.Empty masterTakeoverFinished(
+          com.google.protobuf.RpcController controller,
+          com.orbekk.same.Services.MasterState request)
+          throws com.google.protobuf.ServiceException {
+        return (com.orbekk.same.Services.Empty) channel.callBlockingMethod(
+          getDescriptor().getMethods().get(4),
+          controller,
+          request,
+          com.orbekk.same.Services.Empty.getDefaultInstance());
+      }
+      
     }
   }
   
@@ -7902,40 +8110,43 @@ public final class Services {
       "\n\004data\030\002 \002(\t\022\020\n\010revision\030\003 \002(\003\"u\n\013Master" +
       "State\022\022\n\nmaster_url\030\001 \001(\t\022\021\n\tmaster_id\030\002" +
       " \001(\005\022\024\n\014network_name\030\003 \001(\t\022\027\n\017master_loc" +
-      "ation\030\004 \001(\t\022\020\n\010revision\030\005 \001(\003\"8\n\026MasterT" +
-      "akeoverResponse\022\036\n\026highest_known_revisio" +
-      "n\030\001 \001(\003\"T\n\021FullStateResponse\022\020\n\010revision" +
-      "\030\001 \001(\003\022-\n\tcomponent\030\002 \003(\0132\032.com.orbekk.s",
-      "ame.Component\">\n\013ClientState\022\013\n\003url\030\001 \001(" +
-      "\t\022\020\n\010location\030\002 \001(\t\022\020\n\010revision\030\003 \001(\003\"A\n" +
-      "\020NetworkDirectory\022-\n\007network\030\001 \003(\0132\034.com" +
-      ".orbekk.same.MasterState\"T\n\014PaxosRequest" +
-      "\022,\n\006client\030\001 \001(\0132\034.com.orbekk.same.Clien" +
-      "tState\022\026\n\016proposalNumber\030\002 \001(\005\"\037\n\rPaxosR" +
-      "esponse\022\016\n\006result\030\001 \001(\0052\261\002\n\006Client\022>\n\010Se" +
-      "tState\022\032.com.orbekk.same.Component\032\026.com" +
-      ".orbekk.same.Empty\022B\n\nMasterDown\022\034.com.o" +
-      "rbekk.same.MasterState\032\026.com.orbekk.same",
-      ".Empty\022W\n\016MasterTakeover\022\034.com.orbekk.sa" +
-      "me.MasterState\032\'.com.orbekk.same.MasterT" +
-      "akeoverResponse\022J\n\014GetFullState\022\026.com.or" +
-      "bekk.same.Empty\032\".com.orbekk.same.FullSt" +
-      "ateResponse2\260\001\n\006Master\022J\n\022JoinNetworkReq" +
-      "uest\022\034.com.orbekk.same.ClientState\032\026.com" +
-      ".orbekk.same.Empty\022Z\n\022UpdateStateRequest" +
-      "\022\032.com.orbekk.same.Component\032(.com.orbek" +
-      "k.same.UpdateComponentResponse2\236\001\n\tDirec" +
-      "tory\022G\n\017RegisterNetwork\022\034.com.orbekk.sam",
-      "e.MasterState\032\026.com.orbekk.same.Empty\022H\n" +
-      "\013GetNetworks\022\026.com.orbekk.same.Empty\032!.c" +
-      "om.orbekk.same.NetworkDirectory2\241\001\n\005Paxo" +
-      "s\022H\n\007Propose\022\035.com.orbekk.same.PaxosRequ" +
-      "est\032\036.com.orbekk.same.PaxosResponse\022N\n\rA" +
-      "cceptRequest\022\035.com.orbekk.same.PaxosRequ" +
-      "est\032\036.com.orbekk.same.PaxosResponse2Y\n\rS" +
-      "ystemService\022H\n\017GetSystemStatus\022\026.com.or" +
-      "bekk.same.Empty\032\035.com.orbekk.same.System" +
-      "StatusB\003\210\001\001"
+      "ation\030\004 \001(\t\022\020\n\010revision\030\005 \001(\003\"]\n\026MasterT" +
+      "akeoverResponse\022\017\n\007success\030\002 \001(\010\0222\n\014clie" +
+      "nt_state\030\003 \001(\0132\034.com.orbekk.same.ClientS" +
+      "tate\"T\n\021FullStateResponse\022\020\n\010revision\030\001 ",
+      "\001(\003\022-\n\tcomponent\030\002 \003(\0132\032.com.orbekk.same" +
+      ".Component\">\n\013ClientState\022\013\n\003url\030\001 \001(\t\022\020" +
+      "\n\010location\030\002 \001(\t\022\020\n\010revision\030\003 \001(\003\"A\n\020Ne" +
+      "tworkDirectory\022-\n\007network\030\001 \003(\0132\034.com.or" +
+      "bekk.same.MasterState\"T\n\014PaxosRequest\022,\n" +
+      "\006client\030\001 \001(\0132\034.com.orbekk.same.ClientSt" +
+      "ate\022\026\n\016proposalNumber\030\002 \001(\005\"\037\n\rPaxosResp" +
+      "onse\022\016\n\006result\030\001 \001(\0052\201\003\n\006Client\022>\n\010SetSt" +
+      "ate\022\032.com.orbekk.same.Component\032\026.com.or" +
+      "bekk.same.Empty\022B\n\nMasterDown\022\034.com.orbe",
+      "kk.same.MasterState\032\026.com.orbekk.same.Em" +
+      "pty\022W\n\016MasterTakeover\022\034.com.orbekk.same." +
+      "MasterState\032\'.com.orbekk.same.MasterTake" +
+      "overResponse\022J\n\014GetFullState\022\026.com.orbek" +
+      "k.same.Empty\032\".com.orbekk.same.FullState" +
+      "Response\022N\n\026MasterTakeoverFinished\022\034.com" +
+      ".orbekk.same.MasterState\032\026.com.orbekk.sa" +
+      "me.Empty2\260\001\n\006Master\022J\n\022JoinNetworkReques" +
+      "t\022\034.com.orbekk.same.ClientState\032\026.com.or" +
+      "bekk.same.Empty\022Z\n\022UpdateStateRequest\022\032.",
+      "com.orbekk.same.Component\032(.com.orbekk.s" +
+      "ame.UpdateComponentResponse2\236\001\n\tDirector" +
+      "y\022G\n\017RegisterNetwork\022\034.com.orbekk.same.M" +
+      "asterState\032\026.com.orbekk.same.Empty\022H\n\013Ge" +
+      "tNetworks\022\026.com.orbekk.same.Empty\032!.com." +
+      "orbekk.same.NetworkDirectory2\241\001\n\005Paxos\022H" +
+      "\n\007Propose\022\035.com.orbekk.same.PaxosRequest" +
+      "\032\036.com.orbekk.same.PaxosResponse\022N\n\rAcce" +
+      "ptRequest\022\035.com.orbekk.same.PaxosRequest" +
+      "\032\036.com.orbekk.same.PaxosResponse2Y\n\rSyst",
+      "emService\022H\n\017GetSystemStatus\022\026.com.orbek" +
+      "k.same.Empty\032\035.com.orbekk.same.SystemSta" +
+      "tusB\003\210\001\001"
     };
     com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
       new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -7987,7 +8198,7 @@ public final class Services {
           internal_static_com_orbekk_same_MasterTakeoverResponse_fieldAccessorTable = new
             com.google.protobuf.GeneratedMessage.FieldAccessorTable(
               internal_static_com_orbekk_same_MasterTakeoverResponse_descriptor,
-              new java.lang.String[] { "HighestKnownRevision", },
+              new java.lang.String[] { "Success", "ClientState", },
               com.orbekk.same.Services.MasterTakeoverResponse.class,
               com.orbekk.same.Services.MasterTakeoverResponse.Builder.class);
           internal_static_com_orbekk_same_FullStateResponse_descriptor =
diff --git a/same/src/main/java/com/orbekk/same/services.proto b/same/src/main/java/com/orbekk/same/services.proto
index 1f0064f..8c05e3f 100644
--- a/same/src/main/java/com/orbekk/same/services.proto
+++ b/same/src/main/java/com/orbekk/same/services.proto
@@ -38,8 +38,10 @@ message MasterState {
     optional int64 revision = 5;
 }
 
+// Next tag: 4
 message MasterTakeoverResponse {
-	optional int64 highest_known_revision = 1;
+	optional bool success = 2;
+	optional ClientState client_state = 3;
 }
 
 message FullStateResponse {
@@ -71,6 +73,7 @@ service Client {
     rpc MasterDown (MasterState) returns (Empty);
     rpc MasterTakeover (MasterState) returns (MasterTakeoverResponse);
     rpc GetFullState (Empty) returns (FullStateResponse);
+    rpc MasterTakeoverFinished (MasterState) returns (Empty);
 }
 
 service Master {