eoserv-pubfiles.diff

diff --git a/config/files.ini b/config/files.ini
index e4761da..93e3405 100644
--- a/config/files.ini
+++ b/config/files.ini
@@ -11,6 +11,11 @@ ENF = ./data/pub/dtn001.enf
 ESF = ./data/pub/dsl001.esf
 ECF = ./data/pub/dat001.ecf
 
+## AutoSplitPubFiles (bool)
+# Automatically split large pub files in to multiple smaller files
+# This is required for more than 900 items to work
+AutoSplitPubFiles = yes
+
 ## NewsFile (string)
 # File containing the MotD and news
 NewsFile = ./data/news.txt
diff --git a/src/eoclient.cpp b/src/eoclient.cpp
index 4d5194a..27248a3 100644
--- a/src/eoclient.cpp
+++ b/src/eoclient.cpp
@@ -76,19 +76,19 @@ void EOClient::Tick()
 
 		if (upload_available != 0)
 		{
-			upload_available = std::fread(&this->send_buffer[this->send_buffer_ppos + 1], 1, upload_available, this->upload_fh);
+			upload_available = std::fread(&this->send_buffer[this->send_buffer_ppos], 1, upload_available, this->upload_fh);
 
 			// Dynamically rewrite the bytes of the map to enable PK
 			if (this->upload_type == FILE_MAP && this->server()->world->config["GlobalPK"] && !this->server()->world->PKExcept(player->character->mapid))
 			{
 				if (this->upload_pos <= 0x03 && this->upload_pos + upload_available > 0x03)
-					this->send_buffer[this->send_buffer_ppos + 1 + 0x03 - this->upload_pos] = 0xFF;
+					this->send_buffer[this->send_buffer_ppos + 0x03 - this->upload_pos] = 0xFF;
 
 				if (this->upload_pos <= 0x03 && this->upload_pos + upload_available > 0x04)
-					this->send_buffer[this->send_buffer_ppos + 1 + 0x04 - this->upload_pos] = 0x01;
+					this->send_buffer[this->send_buffer_ppos + 0x04 - this->upload_pos] = 0x01;
 
 				if (this->upload_pos <= 0x1F && this->upload_pos + upload_available > 0x1F)
-					this->send_buffer[this->send_buffer_ppos + 1 + 0x1F - this->upload_pos] = 0x04;
+					this->send_buffer[this->send_buffer_ppos + 0x1F - this->upload_pos] = 0x04;
 			}
 
 			this->upload_pos += upload_available;
@@ -294,21 +294,59 @@ void EOClient::Execute(const std::string &data)
 
 bool EOClient::Upload(FileType type, int id, InitReply init_reply)
 {
-	char mapbuf[7];
-	std::sprintf(mapbuf, "%05i", int(std::abs(id)));
+	bool auto_split = server()->world->config["AutoSplitPubFiles"];
 
-	switch (type)
+	if (type != FILE_MAP && id != 1 && !auto_split)
+		return false;
+
+	this->upload_file_id = id;
+
+	std::string filename;
+	std::size_t file_start = 0;
+	std::size_t file_length = 0;
+
+	std::vector<std::size_t>* file_splits = nullptr;
+
+	if (type == FILE_MAP)
 	{
-		case FILE_MAP: return EOClient::Upload(type, std::string(server()->world->config["MapDir"]) + mapbuf + ".emf", init_reply);
-		case FILE_ITEM: return EOClient::Upload(type, std::string(this->server()->world->config["EIF"]), init_reply);
-		case FILE_NPC: return EOClient::Upload(type, std::string(this->server()->world->config["ENF"]),init_reply);
-		case FILE_SPELL: return EOClient::Upload(type, std::string(this->server()->world->config["ESF"]), init_reply);
-		case FILE_CLASS: return EOClient::Upload(type, std::string(this->server()->world->config["ECF"]), init_reply);
-		default: return false;
+		char mapbuf[7];
+		std::sprintf(mapbuf, "%05i", int(std::abs(id)));
+		filename = std::string(server()->world->config["MapDir"]) + mapbuf + ".emf";
 	}
+	else if (type == FILE_ITEM)
+	{
+		file_splits = &this->server()->world->eif->file_splits;
+		filename = this->server()->world->config["EIF"].GetString();
+	}
+	else if (type == FILE_NPC)
+	{
+		file_splits = &this->server()->world->enf->file_splits;
+		filename = this->server()->world->config["ENF"].GetString();
+	}
+	else if (type == FILE_SPELL)
+	{
+		file_splits = &this->server()->world->esf->file_splits;
+		filename = this->server()->world->config["ESF"].GetString();
+	}
+	else if (type == FILE_CLASS)
+	{
+		file_splits = &this->server()->world->ecf->file_splits;
+		filename = this->server()->world->config["ECF"].GetString();
+	}
+
+	if (auto_split)
+	{
+		if (id < 1 || id >= file_splits->size())
+			return false;
+
+		file_start = (*file_splits)[id - 1];
+		file_length = (*file_splits)[id] - file_start;
+	}
+
+	return EOClient::Upload(type, filename, file_start, file_length, init_reply);
 }
 
-bool EOClient::Upload(FileType type, const std::string &filename, InitReply init_reply)
+bool EOClient::Upload(FileType type, const std::string &filename, std::size_t file_start, std::size_t file_length, InitReply init_reply)
 {
 	using std::swap;
 
@@ -320,17 +358,27 @@ bool EOClient::Upload(FileType type, const std::string &filename, InitReply init
 	if (!this->upload_fh)
 		return false;
 
-	if (std::fseek(this->upload_fh, 0, SEEK_END) != 0)
+	// The size of all pub file headers is 10 bytes
+	std::array<char, 10> pub_header_bytes;
+
+	if (file_length == 0)
 	{
-		std::fclose(this->upload_fh);
-		return false;
+		if (std::fseek(this->upload_fh, 0, SEEK_END) != 0)
+		{
+			std::fclose(this->upload_fh);
+			return false;
+		}
+
+		std::fseek(this->upload_fh, 0, SEEK_SET);
+	}
+	else
+	{
+		file_length += pub_header_bytes.size();
 	}
 
 	this->upload_type = type;
 	this->upload_pos = 0;
-	this->upload_size = std::ftell(this->upload_fh);
-
-	std::fseek(this->upload_fh, 0, SEEK_SET);
+	this->upload_size = file_length;
 
 	std::size_t temp_buffer_size = this->send_buffer.size();
 
@@ -353,12 +401,30 @@ bool EOClient::Upload(FileType type, const std::string &filename, InitReply init
 	builder.AddChar(init_reply);
 
 	if (type != FILE_MAP)
-		builder.AddChar(1);
+		builder.AddChar(this->upload_file_id);
 
 	builder.AddSize(this->upload_size);
 
 	Client::Send(builder);
 
+	// Copy the header from dat001 in to higher numbered files
+	if (file_start != 0)
+	{
+		int bytes_read = std::fread(&pub_header_bytes[0], 1, pub_header_bytes.size(), this->upload_fh);
+
+		if (bytes_read < sizeof pub_header_bytes)
+			return false;
+
+		std::copy(pub_header_bytes.begin(), pub_header_bytes.end(), this->send_buffer.begin() + this->send_buffer_ppos);
+
+		this->upload_pos += bytes_read;
+		this->send_buffer_ppos += bytes_read;
+		this->send_buffer_used += bytes_read;
+
+		if (file_start != pub_header_bytes.size())
+			std::fseek(this->upload_fh, file_start, SEEK_SET);
+	}
+
 	return true;
 }
 
@@ -379,8 +445,8 @@ void EOClient::Send(const PacketBuilder &builder)
 
 		for (std::size_t i = 0; i < data.length(); ++i)
 		{
-			this->send_buffer2_ppos = (this->send_buffer2_ppos + 1) & mask;
 			this->send_buffer2[this->send_buffer2_ppos] = data[i];
+			this->send_buffer2_ppos = (this->send_buffer2_ppos + 1) & mask;
 		}
 
 		this->send_buffer2_used += data.length();
diff --git a/src/eoclient.hpp b/src/eoclient.hpp
index 8315e4f..3ee7540 100644
--- a/src/eoclient.hpp
+++ b/src/eoclient.hpp
@@ -82,9 +82,11 @@ class EOClient : public Client
 		EOClient();
 
 		FileType upload_type;
+		int upload_file_id;
 		std::FILE *upload_fh;
 		std::size_t upload_pos;
 		std::size_t upload_size;
 
 		std::string send_buffer2;
 		std::size_t send_buffer2_gpos;
@@ -105,6 +107,11 @@ class EOClient : public Client
 		ClientState state;
 		int login_attempts;
 
+		int next_eif_id = 1;
+		int next_enf_id = 1;
+		int next_esf_id = 1;
+		int next_ecf_id = 1;
+
 		ActionQueue queue;
 
 		PacketState packet_state;
@@ -141,7 +148,7 @@ class EOClient : public Client
 		void Execute(const std::string &data);
 
 		bool Upload(FileType type, int id, InitReply init_reply);
-		bool Upload(FileType type, const std::string &filename, InitReply init_reply);
+		bool Upload(FileType type, const std::string &filename, std::size_t file_start, std::size_t file_length, InitReply init_reply);
 		void Send(const PacketBuilder &packet);
 
 		~EOClient();
diff --git a/src/eodata.cpp b/src/eodata.cpp
index 3e6bf91..87b84dd 100644
--- a/src/eodata.cpp
+++ b/src/eodata.cpp
@@ -43,6 +43,8 @@ void EIF::Read(const std::string& filename)
 	int numobj = PacketProcessor::Number(this->len[0], this->len[1]);
 	SAFE_SEEK(fh, 1, SEEK_CUR);
 
+	this->file_splits.reserve(1 + (numobj / EIF::FILE_MAX_ENTRIES));
+
 	unsigned char namesize;
 	std::string name;
 	char buf[EIF::DATA_SIZE] = {0};
@@ -54,6 +56,9 @@ void EIF::Read(const std::string& filename)
 	{
 		EIF_Data& newdata = this->data[i];
 
+		if ((i - 1) % EIF::FILE_MAX_ENTRIES == 0)
+			this->file_splits.push_back(std::size_t(std::ftell(fh)) - 1);
+
 		namesize = PacketProcessor::Number(namesize);
 		name.resize(namesize);
 		SAFE_READ(&name[0], sizeof(char), namesize, fh);
@@ -112,7 +117,7 @@ void EIF::Read(const std::string& filename)
 
 		newdata.size = static_cast<EIF::Size>(PacketProcessor::Number(buf[57]));
 
-		if (std::fread(static_cast<void *>(&namesize), sizeof(char), 1, fh) != 1)
+		if (i < numobj && std::fread(static_cast<void *>(&namesize), sizeof(char), 1, fh) != 1)
 		{
 			break;
 		}
@@ -123,6 +128,8 @@ void EIF::Read(const std::string& filename)
 		this->data.pop_back();
 	}
 
+	this->file_splits.push_back(std::size_t(std::ftell(fh)));
+
 	Console::Out("%i items loaded.", this->data.size()-1);
 
 	std::fclose(fh);
@@ -174,6 +181,8 @@ void ENF::Read(const std::string& filename)
 	int numobj = PacketProcessor::Number(this->len[0], this->len[1]);
 	SAFE_SEEK(fh, 1, SEEK_CUR);
 
+	this->file_splits.reserve(1 + (numobj / ENF::FILE_MAX_ENTRIES));
+
 	unsigned char namesize;
 	std::string name;
 	char buf[ENF::DATA_SIZE] = {0};
@@ -185,6 +194,9 @@ void ENF::Read(const std::string& filename)
 	{
 		ENF_Data& newdata = this->data[i];
 
+		if ((i - 1) % ENF::FILE_MAX_ENTRIES == 0)
+			this->file_splits.push_back(std::size_t(std::ftell(fh)) - 1);
+
 		namesize = PacketProcessor::Number(namesize);
 		name.resize(namesize);
 		SAFE_READ(&name[0], sizeof(char), namesize, fh);
@@ -210,7 +222,7 @@ void ENF::Read(const std::string& filename)
 
 		newdata.exp = PacketProcessor::Number(buf[36], buf[37]);
 
-		if (std::fread(static_cast<void *>(&namesize), sizeof(char), 1, fh) != 1)
+		if (i < numobj && std::fread(static_cast<void *>(&namesize), sizeof(char), 1, fh) != 1)
 		{
 			break;
 		}
@@ -221,6 +233,8 @@ void ENF::Read(const std::string& filename)
 		this->data.pop_back();
 	}
 
+	this->file_splits.push_back(std::size_t(std::ftell(fh)));
+
 	Console::Out("%i npc types loaded.", this->data.size()-1);
 
 	std::fclose(fh);
@@ -261,6 +275,8 @@ void ESF::Read(const std::string& filename)
 	int numobj = PacketProcessor::Number(this->len[0], this->len[1]);
 	SAFE_SEEK(fh, 1, SEEK_CUR);
 
+	this->file_splits.reserve(1 + (numobj / ESF::FILE_MAX_ENTRIES));
+
 	unsigned char namesize, shoutsize;
 	std::string name, shout;
 	char buf[ESF::DATA_SIZE] = {0};
@@ -273,6 +289,9 @@ void ESF::Read(const std::string& filename)
 	{
 		ESF_Data& newdata = this->data[i];
 
+		if ((i - 1) % ESF::FILE_MAX_ENTRIES == 0)
+			this->file_splits.push_back(std::size_t(std::ftell(fh)) - 2);
+
 		namesize = PacketProcessor::Number(namesize);
 		name.resize(namesize);
 		if (namesize > 0)
@@ -308,12 +327,12 @@ void ESF::Read(const std::string& filename)
 		newdata.accuracy = PacketProcessor::Number(buf[27], buf[28]);
 		newdata.hp = PacketProcessor::Number(buf[34], buf[35]);
 
-		if (std::fread(static_cast<void *>(&namesize), sizeof(char), 1, fh) != 1)
+		if (i < numobj && std::fread(static_cast<void *>(&namesize), sizeof(char), 1, fh) != 1)
 		{
 			break;
 		}
 
-		if (std::fread(static_cast<void *>(&shoutsize), sizeof(char), 1, fh) != 1)
+		if (i < numobj && std::fread(static_cast<void *>(&shoutsize), sizeof(char), 1, fh) != 1)
 		{
 			break;
 		}
@@ -324,6 +343,8 @@ void ESF::Read(const std::string& filename)
 		this->data.pop_back();
 	}
 
+	this->file_splits.push_back(std::size_t(std::ftell(fh)));
+
 	Console::Out("%i spells loaded.", this->data.size()-1);
 
 	std::fclose(fh);
@@ -364,6 +385,8 @@ void ECF::Read(const std::string& filename)
 	int numobj = PacketProcessor::Number(this->len[0], this->len[1]);
 	SAFE_SEEK(fh, 1, SEEK_CUR);
 
+	this->file_splits.reserve(1 + (numobj / ECF::FILE_MAX_ENTRIES));
+
 	unsigned char namesize;
 	std::string name;
 	char buf[ECF::DATA_SIZE] = {0};
@@ -375,6 +398,9 @@ void ECF::Read(const std::string& filename)
 	{
 		ECF_Data& newdata = this->data[i];
 
+		if ((i - 1) % ECF::FILE_MAX_ENTRIES == 0)
+			this->file_splits.push_back(std::size_t(std::ftell(fh)) - 1);
+
 		namesize = PacketProcessor::Number(namesize);
 		name.resize(namesize);
 		SAFE_READ(&name[0], sizeof(char), namesize, fh);
@@ -394,7 +420,7 @@ void ECF::Read(const std::string& filename)
 		newdata.con = PacketProcessor::Number(buf[10], buf[11]);
 		newdata.cha = PacketProcessor::Number(buf[12], buf[13]);
 
-		if (std::fread(static_cast<void *>(&namesize), sizeof(char), 1, fh) != 1)
+		if (i < numobj && std::fread(static_cast<void *>(&namesize), sizeof(char), 1, fh) != 1)
 		{
 			break;
 		}
@@ -405,6 +431,8 @@ void ECF::Read(const std::string& filename)
 		this->data.pop_back();
 	}
 
+	this->file_splits.push_back(std::size_t(std::ftell(fh)));
+
 	Console::Out("%i classes loaded.", this->data.size()-1);
 
 	std::fclose(fh);
diff --git a/src/eodata.hpp b/src/eodata.hpp
index a58fe7c..98eb879 100644
--- a/src/eodata.hpp
+++ b/src/eodata.hpp
@@ -179,9 +179,11 @@ class EIF
 		}
 
 		static const int DATA_SIZE = 58;
+		static const int FILE_MAX_ENTRIES = 900;
 		std::array<unsigned char, 4> rid;
 		std::array<unsigned char, 2> len;
 		std::vector<EIF_Data> data;
+		std::vector<std::size_t> file_splits;
 
 		EIF(const std::string& filename) { Read(filename.c_str()); }
 
@@ -253,9 +255,11 @@ class ENF
 		};
 
 		static const int DATA_SIZE = 39;
+		static const int FILE_MAX_ENTRIES = 900;
 		std::array<unsigned char, 4> rid;
 		std::array<unsigned char, 2> len;
 		std::vector<ENF_Data> data;
+		std::vector<std::size_t> file_splits;
 
 		ENF(const std::string& filename) { Read(filename.c_str()); }
 
@@ -329,9 +333,11 @@ class ESF
 		};
 
 		static const int DATA_SIZE = 51;
+		static const int FILE_MAX_ENTRIES = 900;
 		std::array<unsigned char, 4> rid;
 		std::array<unsigned char, 2> len;
 		std::vector<ESF_Data> data;
+		std::vector<std::size_t> file_splits;
 
 		ESF(const std::string& filename) { Read(filename.c_str()); }
 
@@ -374,9 +380,11 @@ class ECF
 {
 	public:
 		static const int DATA_SIZE = 14;
+		static const int FILE_MAX_ENTRIES = 250;
 		std::array<unsigned char, 4> rid;
 		std::array<unsigned char, 2> len;
 		std::vector<ECF_Data> data;
+		std::vector<std::size_t> file_splits;
 
 		ECF(const std::string& filename) { Read(filename.c_str()); }
 
diff --git a/src/eoserv_config.cpp b/src/eoserv_config.cpp
index d5e7495..220e34f 100644
--- a/src/eoserv_config.cpp
+++ b/src/eoserv_config.cpp
@@ -7,7 +7,7 @@
 #include "eoserv_config.hpp"
 
 #include "config.hpp"
- 
+
 #include "console.hpp"
 
 #include <string>
@@ -61,6 +61,7 @@ void eoserv_config_validate_config(Config& config)
 	eoserv_config_default(config, "ENF"                , "./data/pub/dtn001.enf");
 	eoserv_config_default(config, "ESF"                , "./data/pub/dsl001.esf");
 	eoserv_config_default(config, "ECF"                , "./data/pub/dat001.ecf");
+	eoserv_config_default(config, "AutoSplitPubFiles"  , true);
 	eoserv_config_default(config, "NewsFile"           , "./data/news.txt");
 	eoserv_config_default(config, "DropsFile"          , "./data/drops.ini");
 	eoserv_config_default(config, "ShopsFile"          , "./data/shops.ini");
diff --git a/src/handlers/Welcome.cpp b/src/handlers/Welcome.cpp
index bf57006..7b6b126 100644
--- a/src/handlers/Welcome.cpp
+++ b/src/handlers/Welcome.cpp
@@ -384,10 +384,10 @@ void Welcome_Agree(Player *player, PacketReader &reader)
 	switch (file)
 	{
 		case FILE_MAP: result = player->client->Upload(FILE_MAP, player->character->mapid, INIT_FILE_MAP); break;
-		case FILE_ITEM: result = player->client->Upload(FILE_ITEM, 1, INIT_FILE_EIF); break;
-		case FILE_NPC: result = player->client->Upload(FILE_NPC, 1, INIT_FILE_ENF); break;
-		case FILE_SPELL: result = player->client->Upload(FILE_SPELL, 1, INIT_FILE_ESF); break;
-		case FILE_CLASS: result = player->client->Upload(FILE_CLASS, 1, INIT_FILE_ECF); break;
+		case FILE_ITEM: result = player->client->Upload(FILE_ITEM, player->client->next_eif_id++, INIT_FILE_EIF); break;
+		case FILE_NPC: result = player->client->Upload(FILE_NPC, player->client->next_enf_id++, INIT_FILE_ENF); break;
+		case FILE_SPELL: result = player->client->Upload(FILE_SPELL, player->client->next_esf_id++, INIT_FILE_ESF); break;
+		case FILE_CLASS: result = player->client->Upload(FILE_CLASS, player->client->next_ecf_id++, INIT_FILE_ECF); break;
 		default: return;
 	}
 
diff --git a/src/socket.cpp b/src/socket.cpp
index 87b5048..498f8e5 100644
--- a/src/socket.cpp
+++ b/src/socket.cpp
@@ -367,8 +367,8 @@ void Client::Send(const std::string &data)
 
 	for (std::size_t i = 0; i < data.length(); ++i)
 	{
-		this->send_buffer_ppos = (this->send_buffer_ppos + 1) & mask;
 		this->send_buffer[this->send_buffer_ppos] = data[i];
+		this->send_buffer_ppos = (this->send_buffer_ppos + 1) & mask;
 	}
 
 	this->send_buffer_used += data.length();
@@ -415,8 +415,8 @@ bool Client::DoSend()
 	std::size_t to_send;
 	for (to_send = 0; to_send < std::min(this->send_buffer_used, sizeof(buf)); ++to_send)
 	{
-		this->send_buffer_gpos = (this->send_buffer_gpos + 1) & mask;
 		buf[to_send] = this->send_buffer[this->send_buffer_gpos];
+		this->send_buffer_gpos = (this->send_buffer_gpos + 1) & mask;
 	}
 
 	const int written = send(this->impl->sock, buf, to_send, 0);