NS3 Network Coding

38Nodes_3.ffm

3
31 32 33 34 35 36 37
10 12 16 29 30
6 4 3 2 7 7 1
0 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 1 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 0 1
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 1
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

NC_NS3.cc

/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */

#include "ns3/core-module.h"
#include "ns3/network-module.h"
#include "ns3/csma-module.h"
#include "ns3/internet-module.h"
#include "ns3/point-to-point-module.h"
#include "ns3/applications-module.h"
#include "ns3/ipv4-global-routing-helper.h"
#include "ns3/netanim-module.h"
#include "my-tag.h"
#include "udp-echo-helper-r.h"
#include <vector>
#include <fstream>
#include <sstream>
#include <regex>
#include <string>
#include <algorithm>
#include <iostream>
#include <sys/stat.h>

using namespace ns3;

NS_LOG_COMPONENT_DEFINE ("NetworkCoding");

int MaxFlow; // Max flow of the current simulation, read from the first line of the input file
std::vector<std::vector<int>> ffGraph;
std::vector<int> destNodes;
std::vector<int> codNodes;
std::vector<int> srcPackages;

NodeContainer ncNodes; //nodos del grafo
NodeContainer tNodes;  //nodos terminales que se añaden a Source y Dests.

std::vector<std::vector<NetDeviceContainer>> ncDevices;
std::vector<NetDeviceContainer> tDevices;

std::vector<std::vector<Ipv4InterfaceContainer>> ncInterfaces;
std::vector<Ipv4InterfaceContainer> tInterfaces;

std::string dirPath;

Ipv4Address multicastSource;
Ipv4Address multicastGroup ("225.1.2.4");
uint16_t multicastPort = 9;

std::vector<std::vector<std::string>> analysis;

int getElemPosV(std::vector<int> V, int elem){
	return (find(V.begin(), V.end(), elem)) - V.begin();
}

uint8_t pktCount = 0;
// Adds a tag to each one of the packets that are received at the source node from the Packet Generator.
// The tags are sequential and wrap around once they reach the max flow value.
void SourceUdpTxTrace(Ptr<const Packet> pkt){
	//idTag -> n = flow
    // Calculate the tag, they'll be sequential powers of two in the interval [1, 1 << MaxFlow]
	uint8_t tag = 1 << (pktCount % MaxFlow);
	pktCount = pktCount + 1;

	MyTag idTag;
	idTag.SetSimpleValue(tag);
	pkt -> AddPacketTag(idTag);
}


// Returns whether this tag is a coded (composite) or uncoded packet.
bool IsCodedTag(uint8_t tag) {
    bool bit_set = false;
    // Check each of the bits, see if more than one is set
    for (unsigned i = 0; i < 8 * sizeof(tag); ++i) {
        if (tag & (1 << i)) {
            if (bit_set)
                return true;
            bit_set = true;
        }
    }
    return false;
}

// Structure that represents a packet that was received on a node.
struct TaggedPacket {
    uint8_t tag;
    long int data;
};

std::vector<TaggedPacket> srcRxPkt;
bool srcFirst = true;
std::vector<bool> source_used_interfaces; // Whether the interface i already sent its packet this burst.
uint32_t srcAIndex = 0;
void SourceSinkRxTrace(Ptr<const Packet> pkt, const Address &addr){
	std::ofstream srcLog;
	if(srcFirst){
		srcLog.open(dirPath+"/source.plg", std::ofstream::trunc);
    srcLog << "Paquetes enviados de Source:" << std::endl << std::endl;
		srcFirst = false;
        source_used_interfaces.resize(srcPackages.size());
	}else{
		srcLog.open(dirPath+"/source.plg", std::ofstream::app);
	}

	analysis.push_back(std::vector<std::string>(2 + destNodes.size(),""));

	//Read the packet tag
	uint8_t pktC;
	MyTag idTag;
	if(pkt -> PeekPacketTag(idTag)){
		pktC = idTag.GetSimpleValue();
	}

	//pktContent
	uint8_t* buffer = new uint8_t[pkt->GetSize()];
	pkt -> CopyData (buffer, pkt->GetSize());
    // Add the packet to the list of received packets.
    TaggedPacket pkt_;
    pkt_.tag = pktC;
    pkt_.data = atol((char*)buffer);
	srcRxPkt.push_back(pkt_);
    // Log the packet data
	srcLog << pkt_.data << " Tag: " << +pkt_.tag << std::endl;
    std::cout << "SourceRecv: " << pkt_.data << " Tag: " << +pkt_.tag << std::endl;

    double timeNow = Simulator::Now().GetSeconds();
    analysis.at(srcAIndex).at(0) = std::to_string(pkt_.data);
	analysis.at(srcAIndex).at(1) = std::to_string(timeNow);
	srcAIndex = srcAIndex + 1;

	Ptr<Socket> srcSocket = Socket::CreateSocket(ncNodes.Get(0), UdpSocketFactory::GetTypeId());
	srcSocket -> Connect(InetSocketAddress(multicastGroup, multicastPort));

	std::ostringstream msgx;
	MyTag outTag;
	long int pktSrc = srcRxPkt.back().data;
	outTag.SetSimpleValue(pktC);

	msgx << pktSrc;
	Ptr<Packet> outPkt = Create<Packet>((uint8_t*) msgx.str().c_str(), pkt -> GetSize());
	outPkt -> AddPacketTag(outTag);

    // When receiving a new burst, reset the used interfaces list.
    if (pktC == 1) {
        source_used_interfaces.clear();
        source_used_interfaces.resize(srcPackages.size());
    }

    // Send the uncoded packets out the correct interfaces
	Ptr<Node> src = ncNodes.Get(0);
	Ipv4StaticRoutingHelper multicast;
	for(uint32_t i = 0; i < srcPackages.size(); i++){
        uint8_t out_tag = (uint8_t)srcPackages.at(i);
		if (out_tag == pktC){
			Ptr<NetDevice> outIf = ncDevices.at(0).at(i + 1).Get(0);
		    multicast.SetDefaultMulticastRoute (src, outIf);
			srcSocket -> Send(outPkt, 0);
            std::cout << "Output to node " << (i + 2) << " tag " << +out_tag << " value " << pktSrc << std::endl;
		} else {
            // if this interface is supposed to output a coded packet, 
            // search the list of received packets for the latest received packets that can form the needed tag.
            if (!source_used_interfaces[i] && IsCodedTag(out_tag)) {
                uint8_t composite_tag = 0;
                long int pkt_data = 0;
                for (int j = srcRxPkt.size() - 1; j >= 0; --j) {
                    const TaggedPacket& stored_pkt = srcRxPkt.at(j);
                    
                    if ((stored_pkt.tag & out_tag) == stored_pkt.tag) {
                        composite_tag |= stored_pkt.tag;
                        pkt_data ^= stored_pkt.data;
                    }
                    // Do not cross the message boundaries.
                    // Message boundaries are defined as the end of a stream of packets with ascending tags
                    // 1 2 4 BOUNDARY 1 2 4 BOUNDARY 1 2 4.
                    // TODO: Packets may arrive out of order?
                    if (stored_pkt.tag == 1) {
                        break;
                    }
                }
                // See if we were able to construct the entire packet
                if (composite_tag == out_tag) {
                    // Send the packet out this interface and mark it as sent so we don't send it again next time.
                    std::ostringstream msgx_;
                    MyTag tag;
                    tag.SetSimpleValue(composite_tag);

                    msgx_ << pkt_data;
                    Ptr<Packet> outPkt_ = Create<Packet>((uint8_t*) msgx_.str().c_str(), pkt -> GetSize());
                    outPkt_ -> AddPacketTag(tag);

                    Ptr<NetDevice> outIf = ncDevices.at(0).at(i + 1).Get(0);
                    multicast.SetDefaultMulticastRoute (src, outIf);
                    srcSocket -> Send(outPkt_, 0);

                    std::cout << "Output to node " << (i + 2) << " tag " << +composite_tag << " value " << pkt_data << std::endl;
                    source_used_interfaces[i] = true;
                }
            }
        }
	}

	srcSocket -> Close();

	Ptr<Ipv4> ipv4Src = src -> GetObject<Ipv4>();
	Ptr<Ipv4StaticRouting> ipv4SR = multicast.GetStaticRouting(ipv4Src);
	ipv4SR -> RemoveMulticastRoute(ipv4SR -> GetNMulticastRoutes() - 1);
}

std::vector<std::vector<std::vector<long int>>> codRxPkt;
std::vector<std::vector<int>> codRxFlags;
std::vector<bool> codFirst;
void CodSinkRxTrace(std::string context, Ptr<const Packet> pkt, const Address &addr){

	std::regex rgx("(\\d+)");
	std::smatch match;
	uint32_t codNode = 0; //codNode that made Callback
    uint32_t codNodeId = 0;
	if(std::regex_search(context, match, rgx)){
		codNodeId =  std::stoi(match[1]);                //codNode #
		codNode = getElemPosV(codNodes, codNodeId); //change to pos in codNodes
	}

    std::ofstream codLog;
    if(codFirst.at(codNode)){
        codLog.open(dirPath+"/codificador_" + std::to_string(codNodeId + 1) +".plg", std::ofstream::trunc);
    codLog << "Trafico de Paquetes en Codificador #" << (codNodeId + 1) << ":" <<
                                                                                                    std::endl << std::endl;
        codFirst.at(codNode) = false;
    }else{
        codLog.open(dirPath+"/codificador_" + std::to_string(codNodeId + 1) +".plg", std::ofstream::app);
    }

	uint8_t pktC; //flag from pkt
	MyTag idTag;
	if(pkt -> PeekPacketTag(idTag)){
		pktC = idTag.GetSimpleValue();
	}

	//pktContent
	uint8_t* buffer = new uint8_t[pkt->GetSize()];
	pkt -> CopyData (buffer, pkt->GetSize());

	int codFlagPos = getElemPosV(codRxFlags.at(codNode), pktC);
	if(codFlagPos == (int)codRxFlags.at(codNode).size()){
		codRxFlags.at(codNode).push_back((int)pktC);
	}
	//save pkt content in one of the vectors of codBuffer
	codRxPkt.at(codNode).at(codFlagPos).push_back(atol((char*)buffer));

	bool checkPkt = true;
	//Checks if there's a stored pkt for each in IF
	for(uint32_t i = 0; i < codRxPkt.at(codNode).size(); i++){
		if(codRxPkt.at(codNode).at(i).size() == 0){ //no hay paquetes con flag i
			checkPkt = false;
		}
	}

	if(checkPkt){
		std::ostringstream msgx;
		MyTag outTag;

        codLog << "Llegan los paquetes: " << std::endl;
		//take and xOr the first packet from each buffer
		long int pktCod = codRxPkt.at(codNode).at(0).at(0);
        codLog << codRxPkt.at(codNode).at(0).at(0) << std::endl;
		codRxPkt.at(codNode).at(0).erase(codRxPkt.at(codNode).at(0).begin());
		uint8_t tag = codRxFlags.at(codNode).at(0);
		for(uint32_t i = 1; i < codRxPkt.at(codNode).size(); i++){
			pktCod = pktCod ^ codRxPkt.at(codNode).at(i).at(0);
            codLog << codRxPkt.at(codNode).at(i).at(0) << std::endl;
			codRxPkt.at(codNode).at(i).erase(codRxPkt.at(codNode).at(i).begin());

			tag = tag ^ codRxFlags.at(codNode).at(i);
		}
		msgx << pktCod;
		outTag.SetSimpleValue(tag);

        codLog << "Sale el paquete: " << std::endl;
        codLog << pktCod << std::endl << std::endl;
        codLog.close();

		Ptr<Node> src = ncNodes.Get(codNodes.at(codNode));

		Ipv4StaticRoutingHelper multicast;
		Ptr<Ipv4> ipv4Src = src -> GetObject<Ipv4>();
		Ptr<Ipv4StaticRouting> ipv4SR = multicast.GetStaticRouting(ipv4Src);

		/*std::vector<uint32_t> outputDevices;
		for(uint32_t i = 0; i < ffGraph.size(); i++){
			if(ffGraph.at(codNodes.at(codNode)).at(i) == 1){
				Ptr<NetDevice> outIf = ncDevices.at(codNodes.at(codNode)).at(i).Get(0);
				std::cout << "outif=" << outIf -> GetIfIndex() << std::endl;
			  outputDevices.push_back(outIf -> GetIfIndex());
			}
		}
		ipv4SR -> AddMulticastRoute (Ipv4Address::GetAny(), Ipv4Address::GetAny(), Ipv4::IF_ANY,
			 													outputDevices);*/

		Ptr<Packet> outPkt = Create<Packet>((uint8_t*) msgx.str().c_str(), pkt -> GetSize());
		outPkt -> AddPacketTag(outTag);
		Ptr<Socket> srcSocket = Socket::CreateSocket(ncNodes.Get(codNodes.at(codNode)),
		 																						UdpSocketFactory::GetTypeId());
		srcSocket -> Connect(InetSocketAddress(multicastGroup, multicastPort));

		for(uint32_t i = 0; i < ffGraph.size(); i++){
			if(ffGraph.at(codNodes.at(codNode)).at(i) == 1){
				Ptr<NetDevice> outIf = ncDevices.at(codNodes.at(codNode)).at(i).Get(0);
				multicast.SetDefaultMulticastRoute (src, outIf);
				srcSocket -> Send(outPkt, 0);
			}
		}

		srcSocket -> Close();

		uint32_t nR = ipv4SR -> GetNMulticastRoutes();
		for(uint32_t i = 0; i < nR; i++){
			ipv4SR -> RemoveMulticastRoute(0);
		}
	}

}

// Performs Gauss-Jordan elimination on the packet tags to decode the packets.
// TODO: This algorithm can be optimized by swapping the places of the packets in the array and working downwards.
void GaussJordan(std::vector<TaggedPacket>& packets) {
    for (int current_msb = 7; current_msb >= 0; --current_msb) {
        for (unsigned i = 0; i < packets.size(); ++i) {
            // Pick the tag with the most significant bit set and use it as pivot
            TaggedPacket& packet = packets[i];
            uint32_t msb_mask = (1 << current_msb);
            uint32_t clear_mask = ~((1 << (current_msb + 1)) - 1);
            // All bits to the left of the MSB must be 0 for it to be picked.
            if ((packet.tag & msb_mask) && (packet.tag & clear_mask) == 0) {
                // XOR this tag with all other packets that have the MSB set.
                for (unsigned j = 0; j < packets.size(); ++j) {
                    if (i == j)
                        continue;
                    TaggedPacket& sub_packet = packets[j];
                    if ((sub_packet.tag & msb_mask) == 0)
                        continue;
                    sub_packet.tag ^= packet.tag;
                    sub_packet.data ^= packet.data;
                }
            }
        }
    }
}

std::vector<std::vector<std::vector<long int>>> destRxPkt;
std::vector<std::vector<int>> destRxFlags;
std::vector<bool> destRouterFirst;
void DestSinkRxTrace(std::string context, Ptr<const Packet> pkt, const Address &addr){
	std::regex rgx("(\\d+)");
	std::smatch match;
	uint32_t destNode = 0; //destNode that made Callback
    uint32_t destNodeId = 0;
	if(std::regex_search(context, match, rgx)){
		destNodeId =  std::stoi(match[1]);                //destNode #
		destNode = getElemPosV(destNodes, destNodeId); //change to pos in destNodes
	}

    std::ofstream log;
    if(destRouterFirst.at(destNode)){
        log.open(dirPath+"/router_dest_" + std::to_string(destNodeId + 1) +".plg", std::ofstream::trunc);
        log << "Trafico de Paquetes en Router destino #" << (destNodeId + 1) << ":" <<
                                                                                                    std::endl << std::endl;
        destRouterFirst.at(destNode) = false;
    }else{
        log.open(dirPath+"/router_dest_" + std::to_string(destNodeId + 1) +".plg", std::ofstream::app);
    }

	uint8_t pktC; //flag from pkt
	MyTag idTag;
	if(pkt -> PeekPacketTag(idTag)){
		pktC = idTag.GetSimpleValue();
	}

	//pktContent
	uint8_t* buffer = new uint8_t[pkt->GetSize()];
	pkt -> CopyData (buffer, pkt->GetSize());

	int destFlagPos = getElemPosV(destRxFlags.at(destNode), pktC);
	if(destFlagPos == (int)destRxFlags.at(destNode).size()){
		destRxFlags.at(destNode).push_back((int)pktC);
	}
	//save pkt content in one of the vectors of destBuffer
	destRxPkt.at(destNode).at(destFlagPos).push_back(atol((char*)buffer));

	bool checkPkt = true;
	//Checks if there's a stored pkt for each in IF
	for(uint32_t i = 0; i < destRxPkt.at(destNode).size(); i++){
		if(destRxPkt.at(destNode).at(i).size() == 0){ //no hay paquetes con flag i
			checkPkt = false;
		}
	}

	if(checkPkt){

		Ptr<Socket> srcSocket = Socket::CreateSocket(ncNodes.Get(destNodes.at(destNode)),
		 																						UdpSocketFactory::GetTypeId());
		srcSocket -> Connect(InetSocketAddress(multicastGroup, multicastPort));


		//if there's no A+B, send first packet of each buffer.
		Ptr<Node> src = ncNodes.Get(destNodes.at(destNode));
		Ipv4StaticRoutingHelper multicast;
        std::vector<TaggedPacket> final_packets;
		for(uint32_t i = 0; i < destRxPkt.at(destNode).size(); i++){
			long int pktdest = destRxPkt.at(destNode).at(i).at(0);
			uint8_t tag = destRxFlags.at(destNode).at(i);
			TaggedPacket packet;
            packet.tag = tag;
            packet.data = pktdest;
            final_packets.push_back(packet);
            log << "Data " << pktdest << " tag " << +tag << std::endl;
		}

        GaussJordan(final_packets);

        // Send the decoded packets to the terminals
        for (unsigned i = 0; i < final_packets.size(); ++i) {
            TaggedPacket& packet = final_packets.at(i);

            std::ostringstream msgx;
            MyTag outTag;

            msgx << packet.data;
            outTag.SetSimpleValue(packet.tag);
            Ptr<Packet> outPkt = Create<Packet>((uint8_t*) msgx.str().c_str(), pkt -> GetSize());
            outPkt -> AddPacketTag(outTag);

            //a Dest only has one outIf to the terminal attached
            Ptr<NetDevice> outIf = tDevices.at(destNode + 1).Get(0);
            multicast.SetDefaultMulticastRoute (src, outIf);
            srcSocket -> Send(outPkt, 0);
        }

		//after sending, erase first pkt from each buffer
		for(uint32_t i = 0; i < destRxPkt.at(destNode).size(); i++){
			destRxPkt.at(destNode).at(i).erase(destRxPkt.at(destNode).at(i).begin());
		}


		srcSocket -> Close();

		Ptr<Ipv4> ipv4Src = src -> GetObject<Ipv4>();
		Ptr<Ipv4StaticRouting> ipv4SR = multicast.GetStaticRouting(ipv4Src);

		uint32_t nR = ipv4SR -> GetNMulticastRoutes();
		for(uint32_t i = 0; i < nR; i++){
			ipv4SR -> RemoveMulticastRoute(0);
		}
	}
}

std::vector<bool> termFirst;
// Mapping of terminal-id -> node-id
std::vector<int> terminal_node_ids;
std::vector<uint32_t> termAIndex;
void TerminalSinkRxTrace(std::string context, Ptr<const Packet> pkt, const Address &addr){
	std::regex rgx("(\\d+)");
	std::smatch match;
	uint32_t termNode = 0; //destNode that made Callback
	if(std::regex_search(context, match, rgx)){
		termNode =  std::stoi(match[1]);                //destNode #
		//id here is tNodes(node - ffgraph.size)
		termNode = termNode - ffGraph.size();
	}

	std::ofstream termLog;
	if(termFirst.at(termNode)){
		termLog.open(dirPath+"/terminal_" + std::to_string(terminal_node_ids[termNode] + 1) +".plg", std::ofstream::trunc);
    termLog << "Paquetes Recibidos por nodo " << (terminal_node_ids[termNode] + 1) << " (Terminal #" << termNode << "):" << 
																									std::endl << std::endl;
		termFirst.at(termNode) = false;
	}else{
		termLog.open(dirPath+"/terminal_" + std::to_string(terminal_node_ids[termNode] + 1) +".plg", std::ofstream::app);
	}

    uint8_t pktC; //flag from pkt
    MyTag idTag;
    if(pkt -> PeekPacketTag(idTag)){
        pktC = idTag.GetSimpleValue();
    }

	//pktContent
	uint8_t* buffer = new uint8_t[pkt->GetSize()];
	pkt -> CopyData (buffer, pkt->GetSize());
	termLog << (atol((char*)buffer)) << " tag " << +pktC << std::endl;
	termLog.close();

	double timeNow = Simulator::Now().GetSeconds();

	analysis.at(termAIndex.at(termNode - 1)).at(1 + termNode) = std::to_string(timeNow);
	termAIndex.at(termNode - 1) = termAIndex.at(termNode - 1) + 1;
}


void simulationAnalysis(std::string dirPath, uint32_t numTuples){
	std::vector<std::vector<double>> pktTravelTime(analysis.size(),
									std::vector<double>(destNodes.size()));
	for(int i = 0; i < MaxFlow; i++){
		double startTime = std::stod(analysis.at(i).at(1));
		for(size_t j = 2; j < analysis.at(i).size(); j++){
			pktTravelTime.at(i).at(j - 2) = std::stod(analysis.at(i).at(j))  - startTime;
		}
	}

	std::ofstream alsLog;
	alsLog.open(dirPath+"/statistics.plg", std::ofstream::trunc);
  alsLog << "Analisis de envío de paquetes (Paquetes enviados: " << numTuples * MaxFlow <<
																								"):" << std::endl << std::endl;

	for (int type = 0; type < MaxFlow; ++type) {
		alsLog << "Tiempo de llegada promedio por destino para paquete tipo :" << (1 << type) << std::endl;
		for(uint32_t i = 0; i <pktTravelTime.at(type).size(); i++){
			alsLog << "Destino #" << i << "(Nodo " << destNodes.at(i) + 1 <<
																	"): " << pktTravelTime.at(type).at(i) << std::endl;
		}
		alsLog << std::endl;
	}

	alsLog << "Tiempo total por destino:" << std::endl;
	for(uint32_t i = 2; i < analysis.back().size(); i++){
		alsLog << "Destino #" << i - 2 << "(Nodo " << destNodes.at(i - 2) + 1 <<
																"): " << std::stod(analysis.back().at(i)) - 1.0 << std::endl;
	}
	alsLog << std::endl;


	alsLog.close();

}

std::vector<int> getMatrixToGraphLevels(std::vector<std::vector<int>> ffGraph){
	std::vector<int> levels (ffGraph.size(), 0);
	levels.at(0) = 1;
	std::vector<int> searchQ;
	searchQ.push_back(0);
	while(!searchQ.empty()){
		int node = searchQ.at(0);
		searchQ.erase(searchQ.begin());

		for(uint32_t i = 0; i < ffGraph.size(); i++){
			if(ffGraph.at(node).at(i) == 1){
				levels.at(i) = levels.at(node) + 1;

				if(getElemPosV(searchQ, i) == (int)searchQ.size()){
					searchQ.push_back(i);
				}
			}
		}
	}
	return levels;
}


int main (int argc, char *argv[]){

	std::cout << "NC-CheckPoint Start" << std::endl;
  CommandLine cmd;
  std::string ffInput;
	uint32_t numTuples;
    uint32_t seed = 0xFFFFFFFF;
  cmd.AddValue ("ffInput", "Name of Input File", ffInput);
	cmd.AddValue ("numTuples", "Number of packet tuples", numTuples);
    cmd.AddValue ("RngSeed", "Random seed", seed);
  cmd.Parse (argc,argv);

    srand(time(NULL));
    if (seed == 0xFFFFFFFF)
        seed = rand();

    ns3::RngSeedManager::SetSeed(seed);

	std::string line;
	std::ifstream Capacity("./scratch/network_coding_pf/NS3Inputs/"+ffInput+".ffm");


	if(Capacity.is_open()){
        // Read the max flow value
        std::getline(Capacity, line);
        MaxFlow = stoi(line);

		std::getline(Capacity, line);
		std::stringstream ss(line);
		std::string item;

    	while(std::getline(ss, item, ' ')){
			destNodes.push_back(stoi(item));
		}

		getline(Capacity, line);
        // Only try to parse this if the line in the file isn't empty
        if (line != "") {
		  std::stringstream ss2(line);
		  while(std::getline(ss2, item, ' ')){
			 codNodes.push_back(stoi(item));
		  }
        }

		std::getline(Capacity, line);
		std::stringstream ss3(line);
		while(std::getline(ss3, item, ' ')){
			srcPackages.push_back(stoi(item));
		}

		while(std::getline(Capacity, line)){
			std::vector<int> col;
			std::stringstream ss4(line);
			while(std::getline(ss4, item, ' ')){
				col.push_back(stoi(item));
			}
			ffGraph.push_back(col);
		}
		Capacity.close();
	}else{
		std::cout << "Unable to open file\n";
	}

	std::cout << "NC-CheckPoint Input read" << std::endl;

  LogComponentEnable ("UdpEchoClientRApplication", LOG_LEVEL_INFO);


  ncNodes.Create (ffGraph.size());

  tNodes.Create(1); //Source
  tNodes.Create(destNodes.size()); //Destinos

  destRouterFirst = std::vector<bool>(destNodes.size(), true);

  PointToPointHelper pointToPoint;
  pointToPoint.SetDeviceAttribute ("DataRate", StringValue ("5Mbps"));
  pointToPoint.SetChannelAttribute ("Delay", StringValue ("2ms"));

	ncDevices = std::vector<std::vector<NetDeviceContainer>>(ffGraph.size(),
	 				std::vector<NetDeviceContainer>(ffGraph.size(), NetDeviceContainer()));
	tDevices = std::vector<NetDeviceContainer>(tNodes.GetN(), NetDeviceContainer());

    terminal_node_ids = std::vector<int>(tNodes.GetN());

  //conexión terminal 0 -> Source
  NodeContainer temp = NodeContainer();
  temp.Add(tNodes.Get(0));
  temp.Add(ncNodes.Get(0));
  tDevices.at(0) = pointToPoint.Install(temp);

  //conexiones del grafo nodo i -> nodo j
  for(uint32_t i = 0; i < ffGraph.size(); i++){
    for(uint32_t j = 0; j < ffGraph.size(); j++){
      if(ffGraph.at(i).at(j) == 1){
        temp = NodeContainer();
        temp.Add(ncNodes.Get(i));
        temp.Add(ncNodes.Get(j));
        ncDevices.at(i).at(j) = pointToPoint.Install(temp);
      }
    }
  }

  //conexiones destino i -> terminal i
  for(uint32_t i = 1; i < tNodes.GetN(); i++){
    temp = NodeContainer();
    temp.Add(ncNodes.Get(destNodes.at(i - 1))); //ncNodes.Get ( #dest )
    temp.Add(tNodes.Get(i));
    terminal_node_ids[i] = destNodes.at(i - 1);
    tDevices.at(i) = pointToPoint.Install(temp);
  }

	std::cout << "NC-CheckPoint Nodes and Devices Created" << std::endl;

  InternetStackHelper stack;
  stack.InstallAll();

  Ipv4AddressHelper address;
  int a = 1;
  int b = 0;
  int c = 0;
  int d = 0;


	ncInterfaces = std::vector<std::vector<Ipv4InterfaceContainer>>(ffGraph.size(),
	             std::vector<Ipv4InterfaceContainer>(ffGraph.size(), Ipv4InterfaceContainer()));
	tInterfaces = std::vector<Ipv4InterfaceContainer>(ffGraph.size(), Ipv4InterfaceContainer());
	//IPs para las redes en el grafo
  for(uint32_t i = 0; i < ffGraph.size(); i++){
    for(uint32_t j = 0; j < ffGraph.size(); j++){
      if(ffGraph.at(i).at(j) == 1){
        std::string addr = "" + std::to_string(a) + "." + std::to_string(b) + "." + std::to_string(c) + "." + std::to_string(d);
        address.SetBase(Ipv4Address(addr.c_str()),"255.255.255.252");
        ncInterfaces.at(i).at(j) = address.Assign (ncDevices.at(i).at(j));

        //todas las conexiones p2p se hicieron en redes con subnetting a 2 hosts. mascara /30
        d = d + 4;
        if(d == 256){
          d = 0;
          c = c + 1;
        }
        if(c == 256){
          c = 0;
          b = b + 1;
        }
        if(b == 256){
          b = 0;
          a = a + 1;
        }
        if(a == 256){
          return 0; //no hay suficientes redes.
        }
      }
    }
  }

  //IPs para las redes entre terminales y nodos source/dest
  //tInterface(0) es el source multicast.
  for(uint32_t i = 0; i < tNodes.GetN(); i++){
    std::string addr = "" + std::to_string(a) + "." + std::to_string(b) + "." + std::to_string(c) + "." + std::to_string(d);
    address.SetBase(addr.c_str(),"255.255.255.252");
    tInterfaces.at(i) = address.Assign (tDevices.at(i));

    d = d + 4;
    if(d == 256){
      d = 0;
      c = c + 1;
    }
    if(c == 256){
      c = 0;
      b = b + 1;
    }
    if(b == 256){
      b = 0;
      a = a + 1;
    }
    if(a == 256){
      return 0; //no hay suficientes redes.
    }
  }

	std::cout << "NC-CheckPoint Ipv4IF SET" << std::endl;


  multicastSource = (tInterfaces.at(0).GetAddress(0));


  Ipv4StaticRoutingHelper multicast;

  //Set up a Default Multicast Route to terminal 0.
  Ptr<Node> sender = tNodes.Get(0);
  Ptr<NetDevice> senderIf = tDevices.at(0).Get(0);
  multicast.SetDefaultMulticastRoute (sender, senderIf);

	std::cout << "NC-CheckPoint multiSource / T0 -> N0 Set" << std::endl;

  //Configure a (static) multicast route on all nodes
  std::vector<int> multiQ;
  //multiQ.push_back(0);
  std::vector<int> lastNQ;
  //lastNQ.push_back(-1);//last revised node, init at -1 because first last is the terminal 0

	for(uint32_t i = 0; i < ffGraph.size(); i++){
		if(ffGraph.at(0).at(i) == 1){
			multiQ.push_back(i);
			lastNQ.push_back(0);
		}
	}

  while(!multiQ.empty()){
    int node = multiQ.at(0);
    multiQ.erase(multiQ.begin());
    int lastNode = lastNQ.at(0);
    lastNQ.erase(lastNQ.begin());
		uint32_t isDes = getElemPosV(destNodes, node);
		uint32_t isCod = getElemPosV(codNodes,  node);

    Ptr<Node> multicastRouter = ncNodes.Get (node);  // The node in question
		Ptr<NetDevice> inputIf;  // The input NetDevice
    if(lastNode == -1){ // if it's node 0, interface is T0 -> N0
      inputIf = tDevices.at(0).Get(1);
    }else{ //else is lastN -> Ni
      inputIf = ncDevices.at(lastNode).at(node).Get(1);
    }
    NetDeviceContainer outputDevices;  // A container of output NetDevices
    for(uint32_t i = 0; i < ffGraph.size(); i++){
      if(ffGraph.at(node).at(i) == 1){
        multiQ.push_back(i);
        lastNQ.push_back(node);
        outputDevices.Add (ncDevices.at(node).at(i).Get(0)); //the output NetDevice
      }
    }
		//Ignores CodNodes and DestNodes
		if(isDes == destNodes.size() && isCod == codNodes.size()){
	    multicast.AddMulticastRoute (multicastRouter, multicastSource,
	                                 multicastGroup, inputIf, outputDevices);
	  }

  }

	std::cout << "NC-CheckPoint regNodes multicast routing Set" << std::endl;

/*
	//Routing from destNodes to the terminals attached to each one
  //Routing from destNodes is actually default, all trafic will go through if to Ti
  for(uint32_t i = 0; i < destNodes.size(); i++){
    int dest = destNodes.at(i);
    Ptr<Node> destNodeR = ncNodes.Get (dest); //Dest Node
    Ptr<NetDevice> destNodeRIF = tDevices.at(i + 1).Get(0); //output IF from DESTi to Ti;
    multicast.SetDefaultMulticastRoute (destNodeR, destNodeRIF);
  }

  //Routing from codNodes is default?
  //don't think so, cod nodes can have multiple out interfaces.
  //actually should be Default but, All trafic will go through ALL OutputIF
  //I think if sink consumes then generate packets with socket.
	for(uint32_t i = 0; i < codNodes.size(); i++){
    int dest = codNodes.at(i);
    Ptr<Node> codNodeR = ncNodes.Get (dest); //Dest Node
		for(uint32_t j = 0; j < ffGraph.size(); j++){
			if(ffGraph.at(dest).at(j) == 1){
				Ptr<NetDevice> codNodeRIF = ncDevices.at(dest).at(j).Get(0); //output IF from CODi to Ni;
		    multicast.SetDefaultMulticastRoute (codNodeR, codNodeRIF);
			}
		}
  }

	std::cout << "NC-CheckPoint dest/cod Nodes multicast routing Set" << std::endl;
*/

  UdpEchoClientRHelper echoClient(multicastGroup, multicastPort, true);
  echoClient.SetAttribute ("MaxPackets", UintegerValue (numTuples * MaxFlow));
  echoClient.SetAttribute ("Interval", TimeValue (Seconds (0.0005)));
  echoClient.SetAttribute ("PacketSize", UintegerValue (128));

  ApplicationContainer clientApp = echoClient.Install (tNodes.Get(0));
  clientApp.Start(Seconds (1.0));
  clientApp.Stop (Seconds (10.0));

	//Callback on UdpEchoClient to set flags before send on T0
	Config::ConnectWithoutContext ("/NodeList/" + std::to_string(tNodes.Get(0)->GetId()) +
                                 "/ApplicationList/*/$ns3::UdpEchoClientR/Tx", MakeCallback (&SourceUdpTxTrace));

	PacketSinkHelper sink ("ns3::UdpSocketFactory",
    InetSocketAddress (Ipv4Address::GetAny (), multicastPort));
	//PacketSinks are installed in Source, Cod and Dest Routers for coding and decoding
	//PacketSinks are installed in terminals to verify.

  //Sink for Source Router.
  ApplicationContainer sinkS = sink.Install (ncNodes.Get(0));
  sinkS.Start (Seconds (1.0));
  sinkS.Stop (Seconds (10.0));
  Config::ConnectWithoutContext ("/NodeList/" + std::to_string(ncNodes.Get(0)->GetId()) +
                                 "/ApplicationList/*/$ns3::PacketSink/Rx", MakeCallback (&SourceSinkRxTrace));

  //Sink for Cod Routers.
	codRxPkt = std::vector<std::vector<std::vector<long int>>>(codNodes.size(),
	 							std::vector<std::vector<long int>>(2, std::vector<long int>()));
	codRxFlags = std::vector<std::vector<int>>(codNodes.size(), std::vector<int>());
    codFirst = std::vector<bool>(codNodes.size(), true);
  for(uint32_t i = 0; i < codNodes.size(); i++){
    ApplicationContainer sinkC = sink.Install (ncNodes.Get(codNodes.at(i)));
    sinkC.Start (Seconds (1.0));
    sinkC.Stop (Seconds (10.0));
    Config::Connect ("/NodeList/" + std::to_string(ncNodes.Get(codNodes.at(i))->GetId()) +
                                   "/ApplicationList/*/$ns3::PacketSink/Rx",
                                  MakeCallback (&CodSinkRxTrace));
  }

  //Sink for Dest Routers.
	destRxPkt = std::vector<std::vector<std::vector<long int>>>(destNodes.size(),
	 							std::vector<std::vector<long int>>(MaxFlow, std::vector<long int>()));
	destRxFlags = std::vector<std::vector<int>>(destNodes.size(), std::vector<int>());
  for(uint32_t  i = 0; i < destNodes.size(); i++){
    ApplicationContainer sinkD = sink.Install (ncNodes.Get(destNodes.at(i)));
    sinkD.Start (Seconds (1.0));
    sinkD.Stop (Seconds (10.0));
    Config::Connect ("/NodeList/" + std::to_string(ncNodes.Get(destNodes.at(i))->GetId()) +
                                   "/ApplicationList/*/$ns3::PacketSink/Rx",
                                  MakeCallback (&DestSinkRxTrace));
  }

  //Sink for Terminals.
	termFirst = std::vector<bool>(tNodes.GetN(), true);
	termAIndex = std::vector<uint32_t>(tNodes.GetN(), 0);
  for(uint32_t i = 1; i < tNodes.GetN(); i++){
    ApplicationContainer sinkT = sink.Install (tNodes.Get(i));
    sinkT.Start (Seconds (1.0));
    sinkT.Stop (Seconds (10.0));
    Config::Connect ("/NodeList/" + std::to_string(tNodes.Get(i)->GetId()) +
                                   "/ApplicationList/*/$ns3::PacketSink/Rx",
                                  MakeCallback (&TerminalSinkRxTrace));
  }

  std::cout << "NC-CheckPoint echoClient, PacketSinks Set" << std::endl;

	std::vector<int> levels = getMatrixToGraphLevels(ffGraph);
	std::vector<int> sizes;
	for(uint32_t i = 0; i < levels.size(); i++){
		if(levels.at(i) > (int)sizes.size()){
			sizes.resize(levels.at(i));
		}
		if(levels.at(i) != 0){
			sizes.at(levels.at(i) - 1) = sizes.at(levels.at(i) - 1) + 1;
		}
	}

	std::vector<int> lCount(sizes.size(), 0);
	lCount.push_back(1);
	AnimationInterface::SetConstantPosition(tNodes.Get(0), 0, 0);
	for(uint32_t i = 0; i < levels.size(); i++){
		int l = levels.at(i);
		if(l != 0){
			int isDes = getElemPosV(destNodes, i);
			int x = 0;
			if(sizes.at(l - 1) % 2 != 0){
				int init = sizes.at(l - 1) / 2;
				x = -50 * init + lCount.at(l - 1) * 50;
			}else{
				int init = sizes.at(l - 1) / 2;
				int t = lCount.at(l - 1) <= init ? -30 - 50 * (init - 1) : 30;
				int c = lCount.at(l - 1) <= init ? lCount.at(l - 1) : lCount.at(l - 1) - init;
				x = t + 50 * c;
			}
			AnimationInterface::SetConstantPosition(ncNodes.Get(i), x, levels.at(i) * 60);
			lCount.at(l - 1) = lCount.at(l - 1) + 1;

			if(isDes != (int)destNodes.size()){
				AnimationInterface::SetConstantPosition(tNodes.Get(lCount.at(lCount.size() - 1)), x - 20,
				 												 levels.at(i) * 60 + 20);
				lCount.at(lCount.size() - 1) = lCount.at(lCount.size() - 1) + 1;
			}
		}
	}
	AnimationInterface  anim("NCAnimation.xml");

	uint32_t nodeRes     = anim.AddResource(
		"/home/cisco/workspace/ns-3-allinone/ns-3-dev/scratch/network_coding_pf/Media/router-blue.png");
	uint32_t srcNodeRes     = anim.AddResource(
		"/home/cisco/workspace/ns-3-allinone/ns-3-dev/scratch/network_coding_pf/Media/router-green.png");
	uint32_t destNodeRes = anim.AddResource(
		"/home/cisco/workspace/ns-3-allinone/ns-3-dev/scratch/network_coding_pf/Media/router-red.png");
  uint32_t codNodeRes  = anim.AddResource(
		"/home/cisco/workspace/ns-3-allinone/ns-3-dev/scratch/network_coding_pf/Media/router-purple.png");
	uint32_t terminalRes = anim.AddResource(
		"/home/cisco/workspace/ns-3-allinone/ns-3-dev/scratch/network_coding_pf/Media/terminal.png");


	for(uint32_t i = 0; i < ncNodes.GetN(); i++){
		anim.UpdateNodeImage(ncNodes.Get(i) -> GetId(), nodeRes);
		if(levels.at(i) != 0){
			anim.UpdateNodeSize(ncNodes.Get(i) -> GetId(), 20, 20);
		}else{
			anim.UpdateNodeSize(ncNodes.Get(i) -> GetId(), 0, 0);
		}
	}

	//Update Src Router Image.
	anim.UpdateNodeImage(ncNodes.Get(0) -> GetId(), srcNodeRes);

	//Update Cod Routers Image.
  for(uint32_t i = 0; i < codNodes.size(); i++){
    anim.UpdateNodeImage(ncNodes.Get(codNodes.at(i)) -> GetId(), codNodeRes);
  }

  //Update for Dest Routers Image.
  for(uint32_t  i = 0; i < destNodes.size(); i++){
    anim.UpdateNodeImage(ncNodes.Get(destNodes.at(i)) -> GetId(), destNodeRes);
  }

  //Sink for Terminals.
  for(uint32_t i = 0; i < tNodes.GetN(); i++){
    anim.UpdateNodeImage(tNodes.Get(i) -> GetId(), terminalRes);
		anim.UpdateNodeSize(tNodes.Get(i) -> GetId(), 20, 20);
		//animation.UpdateNodeColor(t0r0.Get(0), 122, 186, 122);
  }

	std::cout << "NC-CheckPoint animation Set" << std::endl;

	dirPath = "./scratch/network_coding_pf/NS3Outputs/"+ffInput;
	mkdir(dirPath.c_str(), 0777);

	Simulator::ScheduleDestroy(&simulationAnalysis, dirPath, numTuples);

  Simulator::Run ();
  Simulator::Destroy ();
  return 0;
 }