MultiDeviceSimulator/MultiDeviceSimulatorTests/MultiDeviceSimulatorTests.m

   1 //
   2 //  MultiDeviceSimulatorTests.m
   3 //  MultiDeviceSimulatorTests
   4 //
   5 //
   6
   7 #import <XCTest/XCTest.h>
   8 #import <Foundation/Foundation.h>
   9 #import <Foundation/NSXPCConnection_Private.h>
  10 #import <Security/Security.h>
  11
  12 #import "DeviceSimulatorProtocol.h"
  13 #import "MultiDeviceNetworking.h"
  14 #import <objc/runtime.h>
  15
  16 @interface MDDevice : NSObject<DeviceSimulatorProtocol>
  17 @property NSXPCConnection *connection;
  18 @property NSString *name;
  19 - (instancetype)initWithConnection:(NSXPCConnection *)connection;
  20 @end
  21
  22 #pragma clang diagnostic push
  23 #pragma clang diagnostic ignored "-Wprotocol"
  24 @implementation MDDevice
  25
  26 - (instancetype)initWithConnection:(NSXPCConnection *)connection
  27 {
  28     self = [super init];
  29     if (self) {
  30         self.connection = connection;
  31     }
  32     return self;
  33 }
  34
  35 /* Oh, ObjC, you are my friend */
  36 - (void)forwardInvocation:(NSInvocation *)invocation
  37 {
  38     struct objc_method_description desc = protocol_getMethodDescription(@protocol(DeviceSimulatorProtocol), [invocation selector], true, true);
  39     if (desc.name == NULL) {
  40         [super forwardInvocation:invocation];
  41     } else {
  42         __block bool dooooooEeeeetExclamationPoint = true;
  43         NSLog(@"forwarding to [%@]: %s", self.name, sel_getName(desc.name));
  44         id object = [self.connection synchronousRemoteObjectProxyWithErrorHandler:^(NSError * _Nonnull error) {
  45             NSLog(@"peer failed with: %@", error);
  46             dooooooEeeeetExclamationPoint = false;
  47             //abort();
  48         }];
  49         if(dooooooEeeeetExclamationPoint) {
  50             [invocation invokeWithTarget:object];
  51         }
  52     }
  53 }
  54 @end
  55 #pragma clang diagnostic pop
  56
  57
  58 @interface MultiDeviceSimulatorTests : XCTestCase <NSXPCListenerDelegate>
  59 @property NSMutableDictionary<NSString *,MDDevice *> *connections;
  60 @property MultiDeviceNetworking *network;
  61 @property MDDevice *masterDevice;
  62 @property NSMutableArray <MDDevice *> *minionDevices;
  63 @end
  64
  65 static NSString *testInstanceUUID;
  66
  67 @implementation MultiDeviceSimulatorTests
  68
  69 + (void)setUp
  70 {
  71     testInstanceUUID = [[NSUUID UUID] UUIDString];
  72 }
  73
  74 - (void)setUp
  75 {
  76     signal(SIGPIPE, SIG_IGN);
  77     self.connections = [NSMutableDictionary dictionary];
  78     self.network = [[MultiDeviceNetworking alloc] init];
  79
  80     self.minionDevices = [NSMutableArray array];
  81 }
  82
  83 - (void)tearDown
  84 {
  85     __block uint64_t totalUserUsec = 0, totalSysUsec = 0, totalDiskUsage = 0;
  86     NSMutableDictionary *result = [NSMutableDictionary dictionary];
  87
  88     for (NSString *name in self.connections) {
  89         MDDevice *device = self.connections[name];
  90         NSLog(@"device: %@", name);
  91         [device diagnosticsCPUUsage:^(bool success, uint64_t user_usec, uint64_t sys_usec, NSError *error) {
  92             NSLog(@"cpu %@: %d: u:%llu s:%llu", device.name, success, (unsigned long long)user_usec, (unsigned long long)sys_usec);
  93             totalUserUsec += user_usec;
  94             totalSysUsec += sys_usec;
  95             result[[NSString stringWithFormat:@"cpu-%@", name]] = @{ @"user_usec" : @(user_usec), @"system_usec" : @(sys_usec)};
  96         }];
  97         [device diagnosticsDiskUsage:^(bool success, uint64_t usage, NSError *error) {
  98             NSLog(@"disk %@: %d: %llu", device.name, success, (unsigned long long)usage);
  99             totalDiskUsage += usage;
 100             result[[NSString stringWithFormat:@"disk-%@", name]] = @{ @"usage" : @(usage) };
 101         }];
 102     }
 103
 104     for(MDDevice *dev in self.minionDevices) {
 105         [dev sosLeaveCircle:^(bool success, NSError *error) {
 106             ;
 107         }];
 108     }
 109     [self.masterDevice sosLeaveCircle:^(bool success, NSError *error) {
 110         ;
 111     }];
 112
 113     self.minionDevices = NULL;
 114     self.masterDevice = NULL;
 115
 116     result[@"cpu-total"] = @{ @"user_usec" : @(totalUserUsec), @"system_usec" : @(totalSysUsec)};
 117     result[@"disk-total"] = @{ @"disk" : @(totalDiskUsage) };
 118
 119     NSLog(@"Total cpu: u:%llu s:%llu", (unsigned long long)totalUserUsec, (unsigned long long)totalSysUsec);
 120     NSLog(@"Total disk: %llu", (unsigned long long)totalDiskUsage);
 121
 122     /* XXX check for leaks in all devices */
 123     for (NSString *name in self.connections) {
 124         MDDevice *device = self.connections[name];
 125         [device.connection invalidate];
 126     }
 127     self.connections = NULL;
 128     [self.network dumpKVSState];
 129     [self.network dumpCounters];
 130     [self.network disconnectAll];
 131     [self.network clearTestExpectations];
 132     self.network = NULL;
 133
 134     NSData * jsonData = [NSJSONSerialization dataWithJSONObject:result options:0 error:NULL];
 135
 136     [jsonData writeToFile:[NSString stringWithFormat:@"/tmp/test-result-%@", [self name]] atomically:NO];
 137
 138 }
 139
 140 - (void)setupMasterDevice
 141 {
 142     self.masterDevice = [self device:@"ipad" model:@"iPad" version:@"15E143a"];
 143
 144     [self.masterDevice setupSOSCircle:@"user" password:@"foo" complete:^void(bool success, NSError *error) {
 145         XCTAssert(success, "Expect success: %@", error);
 146     }];
 147
 148     [self.masterDevice sosCircleStatus:^void(SOSCCStatus status, NSError *error) {
 149         XCTAssertEqual(status, kSOSCCInCircle, @"expected to be in circle: %@", error);
 150     }];
 151 }
 152
 153 //MARK: - Device logic
 154
 155 - (MDDevice *)device:(NSString *)name model:(NSString *)model version:(NSString *)version {
 156     MDDevice *device = self.connections[name];
 157     if (device != NULL) {
 158         return NULL;
 159     }
 160
 161     NSXPCConnection *conn = [[NSXPCConnection alloc] initWithServiceName:@"com.apple.Security.DeviceSimulator"];
 162     conn.remoteObjectInterface = [NSXPCInterface interfaceWithProtocol:@protocol(DeviceSimulatorProtocol)];
 163     [conn _setUUID:[NSUUID UUID]]; // select a random instance
 164     [conn resume];
 165
 166     device = [[MDDevice alloc] initWithConnection:conn];
 167     device.name = name;
 168
 169     self.connections[name] = device;
 170
 171     [device setDevice:name
 172               version:version
 173                 model:model
 174          testInstance:testInstanceUUID
 175               network:[self.network endpoint]
 176              complete:^(BOOL success) {
 177         if (!success) {
 178             abort();
 179         }
 180     }];
 181     return device;
 182 }
 183
 184 - (bool)addDeviceToCircle: (MDDevice *) dev {
 185     __block bool added = false;
 186     __block NSString *devPeerID = NULL;
 187     __block NSError *localErr = nil;
 188
 189     [dev setupSOSCircle:@"user" password:@"foo" complete:^void(bool success, NSError *error) {
 190         XCTAssert(success, "Expect success: %@", error);
 191         added = success;
 192     }];
 193
 194     if(added) {
 195         [dev sosRequestToJoin:^(bool success, NSString *peerID, NSError *error) {
 196             XCTAssert(success, "Expect success: %@", error);
 197             XCTAssertNotEqual(peerID, NULL, "Expected to find peerID for peer2");
 198             devPeerID = peerID;
 199             added &= success;
 200         }];
 201     }
 202
 203     if(added) {
 204         __block bool done = false;
 205         for(int tries=0; tries < 5; tries++) {
 206             sleep(2);
 207
 208             [self.masterDevice sosApprovePeer:devPeerID complete:^(BOOL success, NSError *error) {
 209                 localErr = [error copy];
 210                 if(success) {
 211                     localErr = nil;
 212                     done = true;
 213                 }
 214             }];
 215             if(done) break;
 216         }
 217         added &= done;
 218     }
 219     XCTAssert(added, "Expect success (for approve of %@): %@", devPeerID, localErr);
 220     return added;
 221 }
 222
 223 - (void)addKeychainItems:(unsigned long)items toDevice:(MDDevice *)device
 224 {
 225     NSDictionary *addItem = @{
 226                               (__bridge id)kSecClass :(__bridge id)kSecClassInternetPassword,
 227                               (__bridge id)kSecValueData : [@"foo" dataUsingEncoding:NSUTF8StringEncoding],
 228                               (__bridge id)kSecAttrAccessGroup: @"com.apple.cfnetwork",
 229                               (__bridge id)kSecAttrSyncViewHint: @"PCS-MasterKey",
 230                               (__bridge id)kSecAttrDescription: @"delete me if found",
 231                               (__bridge id)kSecAttrServer: @"server",
 232                               (__bridge id)kSecAttrAccount: @"account",
 233                               (__bridge id)kSecAttrSynchronizable: @YES,
 234                               (__bridge id)kSecAttrPath: @"/path",
 235                               (__bridge id)kSecAttrIsInvisible: @YES,
 236                               };
 237
 238     [device secItemAdd:addItem complete:^void(OSStatus status, NSDictionary *result) {
 239         NSLog(@"Result string was dev1: %d %@", (int)status, result);
 240         XCTAssertEqual(status, 0, "Expect success");
 241     }];
 242
 243 }
 244
 245 - (void)runSigninWithAdditionalDevices:(unsigned)additionalDeviceCount keychainItems:(unsigned long)items {
 246
 247     for (unsigned n = 0; n < additionalDeviceCount; n++) {
 248         MDDevice *dev = [self device:[NSString stringWithFormat:@"mac-%u", n] model:@"Mac Pro" version:@"17E121"];
 249         if(dev) {
 250             [self addDeviceToCircle: dev];
 251             [self.minionDevices addObject:dev];
 252         }
 253     }
 254
 255     if (items) {
 256         [self addKeychainItems:items toDevice:self.masterDevice];
 257     }
 258 }
 259
 260
 261 - (void)testPref3Devices {
 262     [self setupMasterDevice];
 263
 264     [self measureBlock:^{
 265         [self runSigninWithAdditionalDevices:2 keychainItems:0];
 266     }];
 267 }
 268
 269 #if 0 /* disabled because of 10min time limit in bats (for now) */
 270
 271 - (void)testPref3Devices1 {
 272     [self setupMasterDevice];
 273     [self runSigninWithAdditionalDevices:2 keychainItems:1];
 274     sleep(60);
 275 }
 276
 277 - (void)testPref3Devices10 {
 278     [self setupMasterDevice];
 279     [self runSigninWithAdditionalDevices:2 keychainItems:10];
 280     sleep(60);
 281 }
 282
 283 - (void)testPref3Devices100 {
 284     [self setupMasterDevice];
 285     [self runSigninWithAdditionalDevices:2 keychainItems:100];
 286     sleep(60);
 287 }
 288
 289 - (void)testPref3Devices1000 {
 290     [self setupMasterDevice];
 291     [self runSigninWithAdditionalDevices:2 keychainItems:1000];
 292     sleep(60);
 293     sleep(1);
 294 }
 295
 296 - (void)testPref6Devices {
 297     [self setupMasterDevice];
 298
 299     [self measureBlock:^{
 300         [self runSigninWithAdditionalDevices:5 keychainItems:0];
 301     }];
 302 }
 303
 304 - (void) testDevices6Retired {
 305     [self setupMasterDevice];
 306
 307     [self measureBlock:^{
 308         [self runSigninWithAdditionalDevices:5 keychainItems:0];
 309     }];
 310
 311 }
 312 #endif
 313
 314 - (void)test2Device {
 315
 316     NSLog(@"create devices");
 317     MDDevice *dev1 = [self device:@"ipad" model:@"iPad" version:@"15E143a"];
 318     MDDevice *dev2 = [self device:@"mac" model:@"Mac Pro" version:@"17E121"];
 319
 320     /*
 321      * using PCS-MasterKey for direct syncing during inital sync
 322      */
 323
 324     NSDictionary *addItem = @{
 325                               (__bridge id)kSecClass :(__bridge id)kSecClassInternetPassword,
 326                               (__bridge id)kSecValueData : [@"foo" dataUsingEncoding:NSUTF8StringEncoding],
 327                               (__bridge id)kSecAttrAccessGroup: @"com.apple.cfnetwork",
 328                               (__bridge id)kSecAttrSyncViewHint: @"PCS-MasterKey",
 329                               (__bridge id)kSecAttrDescription: @"delete me if found",
 330                               (__bridge id)kSecAttrServer: @"server",
 331                               (__bridge id)kSecAttrAccount: @"account",
 332                               (__bridge id)kSecAttrSynchronizable: @YES,
 333                               (__bridge id)kSecAttrPath: @"/path",
 334                               (__bridge id)kSecAttrIsInvisible: @YES,
 335                               };
 336
 337     NSDictionary *findItem = @{
 338                                (__bridge id)kSecClass :(__bridge id)kSecClassInternetPassword,
 339                                (__bridge id)kSecAttrAccessGroup: @"com.apple.cfnetwork",
 340                                (__bridge id)kSecAttrServer: @"server",
 341                                (__bridge id)kSecAttrAccount: @"account",
 342                                (__bridge id)kSecAttrSynchronizable: @YES,
 343                                };
 344
 345     [dev1 secItemAdd:addItem complete:^void(OSStatus status, NSDictionary *result) {
 346         NSLog(@"Result string was dev1: %d %@", (int)status, result);
 347         XCTAssertEqual(status, 0, "Expect success");
 348     }];
 349     [dev1 secItemCopyMatching:findItem complete:^(OSStatus status, NSArray<NSDictionary *> *result) {
 350         NSLog(@"Result string was dev1: %d %@", (int)status, result);
 351         XCTAssertEqual(status, 0, "Expect success");
 352     }];
 353
 354     /*
 355      * Setup and validate device 1
 356      */
 357
 358     XCTestExpectation *expection = [self expectationWithDescription:@"expect to create circle"];
 359     [dev1 setupSOSCircle:@"user" password:@"foo" complete:^void(bool success, NSError *error) {
 360         XCTAssert(success, "Expect success: %@", error);
 361         [expection fulfill];
 362     }];
 363     [self waitForExpectationsWithTimeout:5.0 handler:nil];
 364
 365     [dev1 sosCircleStatus:^void(SOSCCStatus status, NSError *error) {
 366         XCTAssertEqual(status, kSOSCCInCircle, @"expected to be in circle: %@", error);
 367     }];
 368     [dev2 sosCircleStatus:^void(SOSCCStatus status, NSError *error) {
 369         XCTAssertEqual(status, kSOSCCError, @"expected to be in error: %@", error);
 370     }];
 371
 372     /*
 373      * Setup and validate device 2
 374      */
 375
 376     expection = [self expectationWithDescription:@"expect to create circle"];
 377     [dev2 setupSOSCircle:@"user" password:@"foo" complete:^void(bool success, NSError *error) {
 378         XCTAssert(success, "Expect success: %@", error);
 379         [expection fulfill];
 380     }];
 381     [self waitForExpectationsWithTimeout:5.0 handler:nil];
 382
 383     [dev1 sosCircleStatus:^void(SOSCCStatus status, NSError *error) {
 384         XCTAssertEqual(status, kSOSCCInCircle, @"expected to be in circle: %@", error);
 385     }];
 386     [dev2 sosCircleStatus:^void(SOSCCStatus status, NSError *error) {
 387         XCTAssertEqual(status, kSOSCCNotInCircle, @"expected to be NOT in circle: %@", error);
 388     }];
 389
 390     NSLog(@"Update all views (dev1)");
 391     expection = [self expectationWithDescription:@"expect circle update"];
 392     expection.assertForOverFulfill = false;
 393     [self.network setTestExpectation:expection forKey:@"oak"];
 394
 395     expection.assertForOverFulfill = false;
 396     [dev1 sosEnableAllViews:^(BOOL success, NSError *error) {
 397         XCTAssert(success, "Expect success: %@", error);
 398     }];
 399
 400     [self waitForExpectationsWithTimeout:5.0 handler:nil];
 401     [self.network clearTestExpectations];
 402
 403     __block NSString *peerID1 = NULL;
 404     [dev1 sosPeerID:^(NSString *peerID) {
 405         XCTAssertNotEqual(peerID, NULL, @"expected to find a peerID for peer1");
 406         peerID1 = peerID;
 407     }];
 408
 409     [dev2 sosPeerID:^(NSString *peerID) {
 410         XCTAssertEqual(peerID, NULL, @"expected to NOT find peerID for peer2");
 411     }];
 412
 413     [dev1 sosPeerID:^(NSString *peerID) {
 414         XCTAssertNotEqual(peerID, NULL, @"expected to find a peerID for peer1");
 415         XCTAssertEqualObjects(peerID1, peerID, "dev1 changed ?");
 416     }];
 417
 418     /*
 419      * Validate second device can request to join
 420      */
 421
 422     expection = [self expectationWithDescription:@"expect circle update"];
 423     expection.assertForOverFulfill = false;
 424     [self.network setTestExpectation:expection forKey:@"oak"];
 425
 426     __block NSString *peerID2 = NULL;
 427     [dev2 sosRequestToJoin:^(bool success, NSString *peerID, NSError *error) {
 428         XCTAssert(success, "Expect success: %@", error);
 429         XCTAssertNotEqual(peerID, NULL, @"expected to find peerID for peer2");
 430         peerID2 = peerID;
 431     }];
 432     [self waitForExpectationsWithTimeout:5.0 handler:nil];
 433     [self.network clearTestExpectations];
 434
 435     /*
 436      * Check that device 2 can't self join
 437      */
 438
 439     expection = [self expectationWithDescription:@"expect device 2 can't self join"];
 440     [dev2 sosApprovePeer:peerID2 complete:^(BOOL success, NSError *error) {
 441         XCTAssert(!success, "Expect failure: %@", error);
 442         [expection fulfill];
 443     }];
 444     [self waitForExpectationsWithTimeout:5.0 handler:nil];
 445
 446     [dev2 sosCircleStatus:^void(SOSCCStatus status, NSError *error) {
 447         XCTAssertEqual(status, kSOSCCRequestPending, @"expected to be pending request: %@", error);
 448     }];
 449
 450     [dev1 sosPeerID:^(NSString *peerID) {
 451         XCTAssertNotEqual(peerID, NULL, @"expected to find a peerID for peer1");
 452         XCTAssertEqualObjects(peerID1, peerID, "dev1 changed ?");
 453     }];
 454
 455     /*
 456      * Approve device 2 and enable all views
 457      */
 458
 459     NSLog(@"approve");
 460     expection = [self expectationWithDescription:@"expect circle update"];
 461     expection.assertForOverFulfill = false;
 462     [self.network setTestExpectation:expection forKey:@"oak"];
 463
 464     [dev1 sosApprovePeer:peerID2 complete:^(BOOL success, NSError *error) {
 465         XCTAssert(success, "Expect success: %@", error);
 466     }];
 467     [self waitForExpectationsWithTimeout:60.0 handler:nil];
 468     [self.network clearTestExpectations];
 469
 470
 471     /*
 472      * Validate device 2 made it into circle and have a peerID
 473      */
 474
 475     [dev1 sosCircleStatus:^void(SOSCCStatus status, NSError *error) {
 476         XCTAssertEqual(status, kSOSCCInCircle, @"expected to be in circle: %@", error);
 477     }];
 478     [dev2 sosCircleStatus:^void(SOSCCStatus status, NSError *error) {
 479         XCTAssertEqual(status, kSOSCCInCircle, @"expected to be in circle: %@", error);
 480     }];
 481
 482     [dev2 sosPeerID:^(NSString *peerID) {
 483         XCTAssertNotEqual(peerID, NULL, @"expected to find a peerID for peer2");
 484         peerID2 = peerID;
 485         //XCTAssertEqualObject(peerID1, peerID2, "expect peerID to be different");
 486     }];
 487
 488     /*
 489      * Enable view for syncing
 490      */
 491
 492     NSString *netID1toID2 = [NSString stringWithFormat:@"ak|%@:%@", peerID1, peerID2];
 493     NSString *netID2toID1 = [NSString stringWithFormat:@"ak|%@:%@", peerID2, peerID1];
 494
 495     expection = [self expectationWithDescription:@"expect traffic from 1->2"];
 496     expection.assertForOverFulfill = false;
 497     [self.network setTestExpectation:expection forKey:netID1toID2];
 498
 499     expection = [self expectationWithDescription:@"expect traffic from 2->1"];
 500     expection.assertForOverFulfill = false;
 501     [self.network setTestExpectation:expection forKey:netID2toID1];
 502
 503     expection = [self expectationWithDescription:@"expect circle update"];
 504     expection.assertForOverFulfill = false;
 505     [self.network setTestExpectation:expection forKey:@"oak"];
 506
 507     /*
 508      * Perform initial sync
 509      */
 510
 511     NSLog(@"initial sync");
 512     expection = [self expectationWithDescription:@"perform initial sync"];
 513     [dev2 sosWaitForInitialSync:^(bool success, NSError *error) {
 514         XCTAssert(success, "Expect success for syncing: %@", error);
 515         [expection fulfill];
 516     }];
 517
 518     /*
 519      *
 520      */
 521
 522     NSLog(@"Update all views (dev2)");
 523     [dev2 sosEnableAllViews:^(BOOL success, NSError *error) {
 524         XCTAssert(success, "Expect success: %@", error);
 525     }];
 526
 527     [self waitForExpectationsWithTimeout:60.0 handler:nil];
 528     [self.network clearTestExpectations];
 529
 530     /*
 531      * check syncing did its thing
 532      */
 533     NSLog(@"SecItemCopyMatching");
 534     [dev1 secItemCopyMatching:findItem complete:^(OSStatus status, NSArray<NSDictionary *> *result) {
 535         NSLog(@"Result string was dev1: %d %@", (int)status, result);
 536         XCTAssertEqual(status, 0, "Expect success");
 537     }];
 538
 539     [dev2 secItemCopyMatching:findItem complete:^(OSStatus status, NSArray<NSDictionary *> *result) {
 540         NSLog(@"Result string was dev2: %d %@", (int)status, result);
 541         XCTAssertEqual(status, 0, "Expect success");
 542     }];
 543
 544     NSLog(@"done");
 545 }
 546
 547 @end