Skip to content

Instantly share code, notes, and snippets.

@KhaosT
Last active April 10, 2021 03:53
Show Gist options
  • Save KhaosT/bd52badef4f9add28ce3f806ede90248 to your computer and use it in GitHub Desktop.
Save KhaosT/bd52badef4f9add28ce3f806ede90248 to your computer and use it in GitHub Desktop.
#include <stdio.h>
#include <fcntl.h>
#include <stdlib.h>
#include <ctype.h>
#include <unistd.h>
#define _GNU_SOURCE 1
#include <string.h>
#include <sys/stat.h> // for mkdir
#include <sys/mman.h> // for mmap
#undef ntohl
#undef ntohs
#define RED "\033[0;31m"
#define M0 "\e[0;30m"
#define CYAN "\e[0;36m"
#define M1 "\e[0;31m"
#define GREY "\e[0;37m"
#define M8 "\e[0;38m"
#define M9 "\e[0;39m"
#define GREEN "\e[0;32m"
#define YELLOW "\e[0;33m"
#define BLUE "\e[0;34m"
#define PINK "\e[0;35m"
#define NORMAL "\e[0;0m"
#ifdef LINUX
typedef unsigned long uint64_t;
typedef unsigned short uint16_t;
extern void *memmem (const void *__haystack, size_t __haystacklen,
const void *__needle, size_t __needlelen);
#endif
/**
* Apple iOS OTA/PBZX expander/unpacker/lister/searcher - by Jonathan Levin,
*
* http://NewOSXBook.com/
*
* Free for anyone (AISE) to use, modify, etc. I won't complain :-), but I'd appreciate a mention
*
* Changelog: 02/08/16 - Replaced alloca with malloc () (full OTAs with too many files would have popped stack..)
*
* 02/17/16 - Increased tolerance for corrupt OTA - can now seek to entry in a file
*
* 08/31/16 - Added search in OTA.
*
* 02/28/18 - It's been a while - and ota now does diff!
* Also tidied up and made neater
*
* 12/03/18 - Added -S to search for string null terminated
*
* The last OTA: (seriously, I'm done :-)
*
* 08/06/19 - Integrated @S1guza's symlink fix (Thanks, man!)
* Added pbzx built-in so you don't have to use pbzx first
* Added multiple file processing, compatible with shell expansion
* Can now ota ...whatever... payload.0?? to iterate over all files!
* Added -H to generate SHA-1 hashes for all ('*') or specific files in OTA
* (SHA-1 code taken from public domain, as was lzma)
* 10/23/20 - Add support for new OTA format used by watchOS 7 and HomePod OS 14.x
*
* To compile: now use attached makefile, since there are lzma dependencies
* Remember to add '-DLINUX' if on Linux
*
*
*/
typedef unsigned int uint32_t;
uint64_t pos = 0;
#ifndef NOSHA
#include "sha1.c"
#endif // NOSHA
#pragma pack(1)
struct entry
{
uint32_t header;
uint16_t header_length;
uint32_t typeMagic;
uint8_t type;
uint32_t stuff;
uint8_t nameLen;
uint8_t whatever;
// Followed by file name
// 0x28 stuff
// Content Length: 0x41 - 2 bytes, 0x42 - 4 bytes,
};
#pragma pack()
extern int ntohl(int);
extern short ntohs(short);
uint32_t
swap32(uint32_t arg)
{
return (ntohl(arg));
}
int g_list = 0;
int g_verbose = 0;
char *g_extract = NULL;
char *g_search = NULL;
char *g_hash = NULL;
int g_nullTerm = 0;
// Since I now diff and use open->mmap(2) on several occasions, refactored
// into its own function
//
void *mmapFile(char *FileName, uint64_t *FileSize)
{
int fd = open (FileName, O_RDONLY);
if (fd < 0) { perror (FileName); exit(1);}
// 02/17/2016 - mmap
struct stat stbuf;
int rc = fstat(fd, &stbuf);
char *mmapped = mmap(NULL, // void *addr,
stbuf.st_size , // size_t len,
PROT_READ, // int prot,
MAP_PRIVATE, // int flags,
fd, // int fd,
0); // off_t offset);
if (mmapped == MAP_FAILED) { perror (FileName); exit(1);}
if (FileSize) *FileSize = stbuf.st_size;
close (fd);
return (mmapped);
}
void hashFile (char *File, char *Name, uint32_t Size, short Perms, char *HashCriteria)
{
if (!HashCriteria) return;
if ((HashCriteria[0] != '*') && ! strstr(Name, HashCriteria)) return ;
#define HASH_SIZE 20
uint8_t Message_Digest[SHA1HashSize];
doSHA1((void*)File, Size, Message_Digest);
int i = 0;
printf("%s (%d bytes): ", Name, Size);
for (i = 0; i < HASH_SIZE; i++)
{
printf("%02X", Message_Digest[i]);
}
printf("\n");
}
void
extractFile (char *File, char *Name, uint32_t Size, short Perms, char *ExtractCriteria)
{
// MAYBE extract file (depending if matches Criteria, or "*").
// You can modify this to include regexps, case sensitivity, what not.
// presently, it's just strstr()
if (!ExtractCriteria) return;
if ((ExtractCriteria[0] != '*') && ! strstr(Name, ExtractCriteria)) return;
uint16_t type = Perms & S_IFMT;
Perms &= ~S_IFMT;
if(type != S_IFREG && type != S_IFLNK)
{
fprintf(stderr, "Unknown file type: %o\n", type);
// return;
}
// Ok. Extract . This is simple - just dump the file contents to its directory.
// What we need to do here is parse the '/' and mkdir(2), etc.
char *dirSep = strchr (Name, '/');
while (dirSep)
{
*dirSep = '\0';
mkdir(Name,0755);
*dirSep = '/';
dirSep+=1;
dirSep = strchr (dirSep, '/');
}
if(type == S_IFLNK)
{
/* @s1guza's support for symlinks! */
/* http://newosxbook.com/forum/viewtopic.php?f=3&t=19513 */
char *target = strndup(File, Size);
if(g_verbose)
{
fprintf(stderr, "Symlinking %s to %s\n", Name, target);
}
symlink(target, Name);
fchmodat(AT_FDCWD, Name, Perms, AT_SYMLINK_NOFOLLOW);
free(target);
}
else {
// at this point we're out of '/'s
// go back to the last /, if any
if (g_verbose)
{
fprintf(stderr, "Dumping %d bytes to %s\n", Size, Name);
}
int fd = open (Name, O_WRONLY| O_CREAT);
fchmod (fd, Perms);
write (fd, File, Size);
close (fd);
}
} // end extractFile
void showPos()
{
fprintf(stderr, "POS is %lld\n", pos);
}
void processFile(char *fileName);
int
main(int argc ,char **argv)
{
char *filename ="p";
int i = 0;
if (argc < 2) {
fprintf (stderr,"Usage: %s [-v] [-l] [...] _filename[s]_ \nWhere: -l: list files in update payload\n"
"Where: [...] is one of:\n"
" -e _file: extract file from update payload (use \"*\" for all files)\n"
" -s _string _file: Look for occurences of _string_ in file\n"
" -S _string _file: Look for occurences of _string_, NULL terminated in file\n"
" -H [_file]: get hash digest of specific file (use \"*\" for all files)\n"
" [-n] -d _file1 _file2: Point out differences between OTA _file1 and _file2\n"
" -n to only diff names\n", argv[0]);
exit(10);
}
int exists = 0;
for (i = 1;
(i < argc -1) && (argv[i][0] == '-');
i++)
{
// This is super quick/dirty. You might want to rewrite with getopt, etc..
if (strcmp(argv[i], "-n") == 0) {
exists++;
}
else
if (strcmp (argv[i], "-l") == 0) { g_list++;}
else
if (strcmp (argv[i] , "-v") == 0) { g_verbose++;}
#ifndef NOSHA
else
if (strcmp(argv[i], "-H") == 0) {
if (i == argc -1) { fprintf(stderr, "-H: Option requires an argument (what to extract)\n");
exit(5); }
g_hash = argv[i+1]; i++;
}
#endif
else
if (strcmp (argv[i], "-e") == 0) {
if (i == argc -1) { fprintf(stderr, "-e: Option requires an argument (what to extract)\n");
exit(5); }
g_extract = argv[i+1]; i++;
}
// Added 08/31/16:
// and modified 12/01/2018
else
if ((strcmp (argv[i], "-s") == 0) || (strcmp (argv[i], "-S") == 0)) {
if (i == argc - 2) { fprintf(stderr, "%s: Option requires an argument (search string)\n", argv[i]);
exit(5); }
g_search = argv[i+1];
if (argv[i][1] == 'S') g_nullTerm++;
i++;
}
else {
fprintf(stderr,"Unknown option: %s\n", argv[i]);
return 1;
}
}
// Another little fix if user forgot filename, rather than try to open
if (argv[argc-1][0] == '-') {
fprintf(stderr,"Must supply filename\n"); exit(5);
}
// Loop over filenames:
for (; i < argc; i++)
{
if (strstr(argv[i],".ecc")) continue;
processFile(argv[i]);
}
}
#define PBZX_MAGIC "pbzx"
char *doPBZX (char *pbzxData, int Size, int *ExtractedSize) {
#ifndef NO_PBZX
#define OUT_BUFSIZE 16*1024*1024 // Largest chunk I've seen is 8MB. This is double that.
char * decompressXZChunk(char *buf, int size, char *Into, int *IntoSize);
uint64_t length = 0, flags = 0;
char *returned = malloc(OUT_BUFSIZE);
int returnedSize = OUT_BUFSIZE;
int available = returnedSize;
int pos = strlen(PBZX_MAGIC);
flags = *((uint64_t *) pbzxData + pos);
// read (fd, &flags, sizeof (uint64_t));
pos += sizeof(uint64_t);
flags = __builtin_bswap64(flags);
// fprintf(stderr,"Flags: 0x%llx\n", flags);
int i = 0;
int off = 0;
int warn = 0 ;
int skipChunk = 0;
int rc = 0;
// 03/09/2016 - Fixed for single chunks (payload.0##) files, e.g. WatchOS
// and for multiple chunks. AAPL changed flags on me..
//
// New OTAs use 0x800000 for more chunks, not 0x01000000.
// 08/06/2019 - dang it. it's not flags - it's uncomp chunk size.
uint64_t totalSize = 0;
uint64_t uncompLen = flags;
while (pos < Size){
i++;
//printf("FLAGS: %llx\n", flags);
// rc= read (fd, &flags, sizeof (uint64_t)); // check retval..
flags = *((uint64_t *) (pbzxData +pos));
pos+= sizeof(uint64_t);
flags = __builtin_bswap64(flags);
//printf("FLAGS: %llx\n", flags);
length = *((uint64_t *) (pbzxData +pos));
//rc = read (fd, &length, sizeof (uint64_t));
pos+= sizeof(uint64_t);
length = __builtin_bswap64(length);
skipChunk = 0; // (i < minChunk);
if (getenv("JDEBUG") != NULL) fprintf(stderr,"Chunk #%d (uncomp: %lld, comp length: %lld bytes) %s\n",i, flags,length, skipChunk? "(skipped)":"");
// Let's ignore the fact I'm allocating based on user input, etc..
//char *buf = malloc (length);
//int bytes = read (fd, buf, length);
char *buf = pbzxData + pos;
pos += length;
// flags = *((uint64_t *) (pbzxData +pos));
#if 0
// 6/18/2017 - Fix for WatchOS 4.x OTA wherein the chunks are bigger than what can be read in one operation
int bytes = length;
int totalBytes = bytes;
while (totalBytes < length) {
// could be partial read
bytes = read (fd, buf +totalBytes, length -totalBytes);
totalBytes +=bytes;
}
#endif
// We want the XZ header/footer if it's the payload, but prepare_payload doesn't have that,
// so just warn.
if (memcmp(buf, "\xfd""7zXZ", 6)) { warn++;
fprintf (stderr, "Warning: Can't find XZ header. Instead have 0x%x(?).. This is likely not XZ data.\n",
(* (uint32_t *) buf ));
// Treat as uncompressed
// UNCOMMENT THIS to handle uncomp XZ too..
// write (1, buf, length);
if (available < length)
{
returnedSize += 10 * OUT_BUFSIZE;
available += 10 * OUT_BUFSIZE;
// Can't use realloc!
char *new = malloc(returnedSize);
if (getenv("JDEBUG") != NULL)printf("REALLOCING from %p to %p ,%x, AVAIL: %x\n", returned, new, returnedSize, available);
if (new) {
memcpy(new, returned, returnedSize - available);
free(returned);
returned = new;
}
else { fprintf(stderr,"ERROR!\n"); exit(1);}
}
memcpy(returned + (returnedSize - available), buf, length);
totalSize += length;
available -= length;
}
else // if we have the header, we had better have a footer, too
{
if (strncmp(buf + length - 2, "YZ", 2)) { warn++; fprintf (stderr, "Warning: Can't find XZ footer at 0x%llx (instead have %x). This is bad.\n",
(length -2),
*((unsigned short *) (buf + length - 2)));
}
// if (1 && !skipChunk)
{
// Uncompress chunk
int chunkExpandedSize = available;
char *ptrTo = returned + (returnedSize - available);
decompressXZChunk(buf, length, returned + (returnedSize - available),&chunkExpandedSize);
// printf("DECOMPRESSING to %p - %p\n", ptrTo , ptrTo + chunkExpandedSize);
totalSize += chunkExpandedSize;
available -= chunkExpandedSize;
if (available < OUT_BUFSIZE)
{
returnedSize += 10 * OUT_BUFSIZE;
available += 10 * OUT_BUFSIZE;
// Can't use realloc!
char *new = malloc(returnedSize);
if (getenv("JDEBUG") != NULL)printf("REALLOCING from %p to %p ,%x, AVAIL: %x\n", returned, new, returnedSize, available);
if (new) {
memcpy(new, returned, returnedSize - available);
free(returned);
returned = new;
}
else { fprintf(stderr,"ERROR!\n"); exit(1);}
}
}
warn = 0;
// free (buf); // Not freeing anymore, @ryandesign :-)
}
}
//printf("Total size: %d\n", totalSize);
*ExtractedSize = totalSize;
if (getenv("JDEBUG") != NULL)
{
int f = open ("/tmp/out1", O_WRONLY |O_CREAT);
write (f, returned, totalSize);
close(f);
}
return (returned);
#else
fprintf(stderr,"Not compiled with PBZX support!\n");
return (NULL);
#endif
} // pbzx
void processFile(char *FileName)
{
int color = (getenv("JCOLOR")!= NULL);
fprintf(stderr, "%sProcessing %s%s\n", color ? RED: "", FileName, color ? NORMAL :"");
//unsigned char buf[4096];
uint64_t fileSize;
uint64_t mappedSize;
char *actualMmapped = mmapFile(FileName, &mappedSize);
fileSize = mappedSize;
if (actualMmapped == MAP_FAILED) { perror (FileName); return ;}
char *mmapped = actualMmapped;
char *extracted = NULL;
// File could be a PBZX :-)
if (memcmp(mmapped, PBZX_MAGIC, strlen(PBZX_MAGIC)) ==0)
{
// DO PBZX first!
int extractedSize = 0;
extracted = doPBZX (mmapped, mappedSize, &extractedSize);
mmapped = extracted;
fileSize = extractedSize;
// printf("EXTRACTED: %p, size: 0x%llx\n",mmapped, fileSize);
}
int i = 0;
struct entry *ent = alloca (sizeof(struct entry));
pos = 0;
while(pos + 3*sizeof(struct entry) < fileSize) {
ent = (struct entry *) (mmapped + pos );
printf("pos: %lld, size: %lld\n", pos, fileSize);
uint64_t sPos = pos;
pos += sizeof(struct entry);
if (ent->header != 0x31414159)
{
fprintf (stderr,"Corrupt entry (0x%x at pos %llu@0x%llx).. skipping\n", ent->header,
pos, (uint64_t)(mmapped+pos));
int skipping = 1;
while (skipping)
{
ent = (struct entry *) (mmapped + pos ) ;
while (ent->header != 0x31414159)
{
// #@$#$%$# POS ISN'T ALIGNED!
pos ++;
ent = (struct entry *) (mmapped + pos ) ;
}
// read rest of entry
int nl = ntohs(ent->nameLen);
if (!nl) {
// fprintf(stderr,"False positive.. skipping %d\n",pos);
pos+=1;
}
else { skipping =0;
pos += sizeof(struct entry); }
if (pos > fileSize) return;
}
}
printf ("header length: %x\n", ent -> header_length);
// fprintf(stdout," Here - ENT at pos %d: %x and 0 marker is %x namelen: %d, fileSize: %d\n", pos, ent->usually_0x210_or_0x110, ent->usually_0x00_00, ntohs(ent->nameLen), size);
uint32_t nameLen = ent->nameLen;
if (nameLen > 0xFF) {
printf ("NAME TOO LONG?!");
}
printf ("NameLength: %x\n", nameLen);
// Get Name (immediately after the entry)
//
// 02/08/2016: Fixed this from alloca() - the Apple jumbo OTAs have so many files in them (THANKS GUYS!!)
// that this would exceed the stack limits (could solve with ulimit -s, or also by using
// a max buf size and reusing same buf, which would be a lot nicer)
// Note to AAPL: Life would have been a lot nicer if the name would have been NULL terminated..
// What's another byte per every file in a huge file such as this?
// char *name = (char *) (mmapped+pos);
char *name = malloc (nameLen+1);
printf ("Offset: %llx\n", pos);
strncpy(name, mmapped+pos , nameLen);
name[nameLen] = '\0';
printf("NAME IS %s\n", name);
uint16_t perms = (S_IRUSR | S_IWUSR | S_IXUSR);
if (ent -> type == 'D') {
pos = sPos + ent -> header_length;
perms |= S_IFDIR;
free(name);
continue;
} else if (ent -> type == 'F') {
// Fallthrough
perms |= S_IFREG;
} else if (ent -> type == 'L') {
perms |= S_IFLNK;
} else {
printf("Unexpected type: %x\n", ent -> type);
free(name);
return;
}
pos += nameLen + 0x27;
while (mmapped[pos] != 'T') {
pos += 1;
}
pos += 1;
uint32_t fileSize = 0;
uint8_t fileSizeType = mmapped[pos];
if (fileSizeType == 0x41) {
uint16_t fSize = *(uint16_t *)(mmapped + pos + 1);
fileSize = fSize;
} else if (fileSizeType == 0x42) {
uint32_t fSize = *(uint32_t *)(mmapped + pos + 1);
fileSize = fSize;
} else {
printf ("Unexpected type: %x\n", fileSizeType);
free(name);
return;
}
printf ("FileSize: %x\n", fileSize);
pos = sPos + ent -> header_length;
if (g_list){
if (g_verbose) {
printf ("Entry @0x%d: Mode: %o Size: %d (0x%x) Namelen: %d Name: ", i,
perms, fileSize, fileSize,
ntohs(ent->nameLen));
}
printf ("%s\n", name);}
// Get size (immediately after the name)
if (fileSize)
{
if (g_extract) { extractFile(mmapped +pos, name, fileSize, perms, g_extract);}
// Added 08/05/19 - Hash
if (g_hash) { hashFile (mmapped +pos, name, fileSize, perms, g_hash); }
// Added 08/31/16 - And I swear I should have this from the start.
// So darn simple and sooooo useful!
if (g_search){
char *found = memmem (mmapped+pos, fileSize, g_search, strlen(g_search) + (g_nullTerm ? 1 : 0));
while (found != NULL)
{
int relOffset = found - mmapped - pos;
fprintf(stdout, "Found in Entry: %s, relative offset: 0x%x (Absolute: %lx)",
name,
relOffset,
found - mmapped);
// 12/01/18
if (g_verbose) {
fputc(':', stdout);
fputc(' ', stdout);
char *begin = found;
int i = 0 ;
#define BACK_LIMIT -20
#define FRONT_LIMIT 20
while(begin[i] && i > BACK_LIMIT) { i--;}
for (;begin +i < found; i++) {
if (isprint(begin[i])) putc (begin[i], stdout); else putc ('.', stdout); }
printf("%s%s%s",RED, g_search, NORMAL);
for (i+= strlen(g_search); begin[i] &&( i < FRONT_LIMIT); i++) {
if (isprint(begin[i])) putc (begin[i], stdout); else putc ('.', stdout); }
}
fprintf(stdout,"\n");
// keep looking..
found = memmem (found + 1, fileSize - relOffset , g_search, strlen(g_search) +( g_nullTerm ? 1: 0));
} // end while
} // end g_search
pos +=fileSize;
}
free(name);
} // Back to loop
if (extracted) { /*printf("FREEing %p\n", extracted);*/ free (extracted);}
munmap(actualMmapped, mappedSize);
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment