Toggle Host AP Mode on Mac OS X, Using undocumented APIs.

hostap.m

#import <CoreWLAN/CoreWLAN.h>
#import <objc/message.h>

int main(int argc, char* argv[]) {
  @autoreleasepool {
    int ch;
    NSString *ssid = nil, *password = nil;

    while((ch = getopt(argc, argv, "s:w:h")) != -1) {
      switch(ch) {
      case 's':
        ssid = [NSString stringWithUTF8String:optarg];
        break;
      case 'w':
        password = [NSString stringWithUTF8String:optarg];
        break;
      case '?':
      case 'h':
      default:
        printf("USAGE: %s [-swh] command\n", argv[0]);
        printf("\nOPTIONS:\n");
        printf("   -s ssid     SSID\n");
        printf("   -w password WEP password\n");
        printf("   -h          Print help\n");
        printf("\nCOMMAND:\n");
        printf("   status      Print interface mode\n");
        printf("   start       Start Host AP mode\n");
        printf("   stop        Stop Host AP mode\n");
        return 0;
      }
    }

    NSString *command = nil;
    if(argv[optind]) {
      command = [NSString stringWithUTF8String:argv[optind]];
    }

    CWInterface *iface = [CWInterface interface];

    if(!command || [command isEqualToString:@"status"]) {
      NSString *mode = nil;
      switch(iface.interfaceMode) {
      case kCWInterfaceModeStation:
        mode = @"Station";
        break;
      case kCWInterfaceModeIBSS:
        mode = @"IBSS";
        break;
      case kCWInterfaceModeHostAP:
        mode = @"HostAP";
        break;
      case kCWInterfaceModeNone:
      default:
        mode = @"None";
      }
      printf("%s\n", [mode UTF8String]);
    } else if([command isEqualToString:@"stop"]) {
      // Stop Host AP mode
      if(getuid() != 0) {
        printf("You may need to run this command with root priviledge.\n");
      }
      objc_msgSend(iface, @selector(stopHostAPMode));
    } else if([command isEqualToString:@"start"]) {
      // Channel = 11(2GHz), Channel width = 20MHz
      CWChannel *channel = [CWChannel alloc];
      objc_msgSend(channel, @selector(initWithInfo:), @{@"CHANNEL": @11, @"CHANNEL_FLAGS": @10});

      // securityType = 2: None, 32: WEP
      unsigned long long securityType = 2;
      if(password) {
        securityType = 32;
      }

      // Start Host AP mode
      NSError *error = nil;
      objc_msgSend(iface,
                   @selector(startHostAPModeWithSSID:securityType:channel:password:error:),
                   [ssid dataUsingEncoding:NSUTF8StringEncoding],
                   securityType,
                   channel,
                   password,
                   &error);
      if(error) {
        printf("%s\n", [error.localizedDescription UTF8String]);
        return 1;
      }
    }

    return 0;
  }
}

Makefile

all: hostap

hostap: hostap.m
	clang -framework Foundation -framework CoreWLAN -o $@ $<

clean:
	rm -rf hostap

MIT-LICENSE.txt

Copyright (C) 2012 Yoshimasa Niwa

Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Ok, see my fork for a version that works with 10.8: https://gist.github.com/4418079

Ah, it was not tested on 10.8 yet. Thanks! and it's really difficult to know the comment event on Github... :(((

Just update this from my recent investigation around this API on OS X 10.9 and later.

Simply saying, this API seems like no longer work without Apple's permission.
If you use this command, you'll see an error.

The operation couldn’t be completed. (com.apple.coreWLAN.error error -6768.)

Which means the caller doesn't have a permission to change WiFi interface to Host AP mode.

I don't have any good solutions for this issue yet, it must be difficult to break this rule since it's like a challenging OS X security sandbox.