Skip to content

Instantly share code, notes, and snippets.

What would you like to do?
/* ************************************************************************
* *** Super Graphing Data Logger ***
* ************************************************************************
* Everett Robinson, December 2016. More at:
* This sketch relies on the SD and ethernet libraries in arduino 1.0 or newer.
* The following extra non standard libraries were also used, and will need to
* be added to the libraries folder:
* - Time:
* - EEPROMAnything:
* If this is your first time setting up this project, please go get the
* EEPROM_config sketch from so that you can
* configure the config struct in the EEPROM memory. Usage of the EEPROM
* is needed to make the project resiliant against a temporary loss of power.
* You must also ensure that you have the HC.htm file in the root directory
* of your SD card, as well as a data directory where the datafiles will be
* stored.
* This sketch combines the functionality of an existing fileserver example
* which can be found at
* with the Datalogger example that comes with the new SD library from 1.0,
* as well as some code from the UdpNtpClient example that cones with the
* ethernet library.
* Added to all of these are some tricks to make it manage and serve up the
* datafiles in conjunction with a page which uses highcharts JS to graph it.
* This is basically accomplished using the arduino by itself. Because I
* actually host the highcharts.js files externally, this is true more in
* theory than in actual practice, but oh well. It should work just fine to
* have the highcharts.js file on the arduino's SD card, though loading the
* page will be painfully slow.
* Some of the code this was derived from may or may not be under a GPL
* licence; I'm not entirely sure. I suppose anyone using this should treat
* it like it is too, but I don't really care too much.
* Also if one intends to use this for commercial applications, it may be
* necessary to purchase a license for Highcharts.
#include <SD.h>
#include <Ethernet.h>
#include <EthernetUdp.h>
#include <SPI.h>
#include <string.h>
#include <Time.h>
#include <EEPROM.h>
#include <EEPROMAnything.h>
#include <avr/pgmspace.h>
/************ ETHERNET STUFF ************/
byte mac[] = { 0x90, 0xA2, 0xDA, 0x00, 0x4C, 0x64 };
//byte ip[] = { 192,168,1, 100 };
EthernetServer server(80);
/************** NTP STUFF ***************/
unsigned int localPort = 8888; // local port to listen for UDP packets
IPAddress timeServer(128, 138, 140, 44);//NIST time server IP address: for more info
const int NTP_PACKET_SIZE= 48; //NTP time stamp is in the first 48 bytes of the message
byte packetBuffer[ NTP_PACKET_SIZE]; //buffer to hold incoming and outgoing packets
EthernetUDP Udp;
const int analogPin = 0;
unsigned long lastIntervalTime = 0; //The time the last measurement occured.
#define MEASURE_INTERVAL 600000 //10 minute intervals between measurements (in ms)
unsigned long newFileTime; //The time at which we should create a new week's file
#define FILE_INTERVAL 604800 //One week worth of seconds
//A structure that stores file config variables from EEPROM
typedef struct{
unsigned long newFileTime; //Keeps track of when a newfile should be made.
char workingFilename[19]; //The path and filename of the current week's file
} configuration;
configuration config; //Actually make our config struct
// Strings stored in flash mem for the Html Header (saves ram)
const char HeaderOK_0[] PROGMEM = "HTTP/1.1 200 OK"; //
const char HeaderOK_1[] PROGMEM = "Content-Type: text/html"; //
const char HeaderOK_2[] PROGMEM = ""; //
// A table of pointers to the flash memory strings for the header
const char* const HeaderOK_table[] PROGMEM = {
// A function for reasy printing of the headers
void HtmlHeaderOK(EthernetClient client) {
char buffer[30]; //A character array to hold the strings from the flash mem
for (int i = 0; i < 3; i++) {
strcpy_P(buffer, (char*)pgm_read_word(&(HeaderOK_table[i])));
client.println( buffer );
// Strings stored in flash mem for the Html 404 Header
const prog_char Header404_0[] PROGMEM = "HTTP/1.1 404 Not Found"; //
const prog_char Header404_1[] PROGMEM = "Content-Type: text/html"; //
const prog_char Header404_2[] PROGMEM = ""; //
const prog_char Header404_3[] PROGMEM = "<h2>File Not Found!</h2>";
// A table of pointers to the flash memory strings for the header
const char* const Header404_table[] PROGMEM = {
// Easy peasy 404 header function
void HtmlHeader404(EthernetClient client) {
char buffer[30]; //A character array to hold the strings from the flash mem
for (int i = 0; i < 4; i++) {
strcpy_P(buffer, (char*)pgm_read_word(&(Header404_table[i])));
client.println( buffer );
void setup() {
pinMode(10, OUTPUT); // set the SS pin as an output (necessary!)
digitalWrite(10, HIGH); // but turn off the W5100 chip!
// see if the card is present and can be initialized:
if (!SD.begin(4)) {
//Serial.println("Card failed, or not present");
// don't do anything more:
//Serial.println("card initialized.");
// The SD card is working, start the server and ethernet related stuff!
if (Ethernet.begin(mac) == 0) {
//Serial.println("Failed to configure Ethernet using DHCP");
// don't do anything more:
//Serial.println("DHCP configured!");
EEPROM_readAnything(0,config); // make sure our config struct is syncd with EEPROM
// A function that takes care of the listing of files for the
// main page one sees when they first connect to the arduino.
// it only lists the files in the /data/ folder. Make sure this
// exists on your SD card.
void ListFiles(EthernetClient client) {
File workingDir ="/data/");
while(true) {
File entry = workingDir.openNextFile();
if (! entry) {
client.print("<li><a href=\"/HC.htm?file=");
// A function to get the Ntp Time. This is used to make sure that the data
// points recorded by the arduino are referenced to some meaningful time
// which in our case is UTC represented as unix time (choosen because it
// works simply with highcharts without too much unecessary computation).
unsigned long getTime(){
sendNTPpacket(timeServer); // send an NTP packet to a time server
// wait to see if a reply is available
if ( Udp.parsePacket() ) {
// We've received a packet, read the data from it,NTP_PACKET_SIZE); // read the packet into the buffer
//the timestamp starts at byte 40 of the received packet and is four bytes,
// or two words, long. First, esxtract the two words:
unsigned long highWord = word(packetBuffer[40], packetBuffer[41]);
unsigned long lowWord = word(packetBuffer[42], packetBuffer[43]);
// combine the four bytes (two words) into a long integer
// this is NTP time (seconds since Jan 1 1900):
unsigned long secsSince1900 = highWord << 16 | lowWord;
// Unix time starts on Jan 1 1970. In seconds, that's 2208988800:
const unsigned long seventyYears = 2208988800UL;
// subtract seventy years:
unsigned long epoch = secsSince1900 - seventyYears;
// return Unix time:
return epoch;
// send an NTP request to the time server at the given address,
// necessary for getTime().
unsigned long sendNTPpacket(IPAddress& address){
// set all bytes in the buffer to 0
memset(packetBuffer, 0, NTP_PACKET_SIZE);
// Initialize values needed to form NTP request
// (see URL above for details on the packets)
packetBuffer[0] = 0b11100011; // LI, Version, Mode
packetBuffer[1] = 0; // Stratum, or type of clock
packetBuffer[2] = 6; // Polling Interval
packetBuffer[3] = 0xEC; // Peer Clock Precision
// 8 bytes of zero for Root Delay & Root Dispersion
packetBuffer[12] = 49;
packetBuffer[13] = 0x4E;
packetBuffer[14] = 49;
packetBuffer[15] = 52;
// all NTP fields have been given values, now
// you can send a packet requesting a timestamp:
Udp.beginPacket(address, 123); //NTP requests are to port 123
// How big our line buffer should be for sending the files over the ethernet.
// 75 has worked fine for me so far.
#define BUFSIZ 75
void loop(){
if ((millis() % lastIntervalTime) >= MEASURE_INTERVAL){ //Is it time for a new measurement?
char dataString[20] = "";
int count = 0;
unsigned long rawTime;
rawTime = getTime();
while((rawTime == 39) && (count < 12)){ //server seems to send 39 as an error code
delay(5000); //we want to retry if this happens. I chose
rawTime = getTime(); //12 retries because I'm stubborn/persistent.
count += 1; //NIST considers retry interval of <4s as DoS
} //attack, so fair warning.
if (rawTime != 39){ //If that worked, and we have a real time
//Decide if it's time to make a new file or not. Files are broken
//up like this to keep loading times for each chart bearable.
//Lots of string stuff happens to make a new filename if necessary.
if (rawTime >= config.newFileTime){
int dayInt = day(rawTime);
int monthInt = month(rawTime);
int yearInt = year(rawTime);
char newFilename[18] = "";
char dayStr[3];
char monthStr[3];
char yearStr[5];
char subYear[3];
if (dayInt < 10){
if (monthInt < 10){
//we only want the last two digits of the year
memcpy( subYear, &yearStr[2], 3 );
//make sure we update our config variables:
config.newFileTime += FILE_INTERVAL;
//Write the changes to EEPROM. Bad things may happen if power is lost midway through,
//but it's a small risk we take. Manual fix with EEPROM_config sketch can correct it.
EEPROM_writeAnything(0, config);
//get the values and setup the string we want to write to the file
int sensor = analogRead(analogPin);
char timeStr[12];
char sensorStr[6];
//open the file we'll be writing to.
File dataFile =, FILE_WRITE);
// if the file is available, write to it:
if (dataFile) {
// print to the serial port too:
// if the file isn't open, pop up an error:
else {
Serial.println("Error opening datafile for writing");
Serial.println("Couldn't resolve a time from the Ntp Server.");
//Update the time of the last measurment to the current timer value
lastIntervalTime = millis();
//No measurements to be made, make sure the webserver is available for connections.
char clientline[BUFSIZ];
int index = 0;
EthernetClient client = server.available();
if (client) {
// an http request ends with a blank line
boolean current_line_is_blank = true;
// reset the input buffer
index = 0;
while (client.connected()) {
if (client.available()) {
char c =;
// If it isn't a new line, add the character to the buffer
if (c != '\n' && c != '\r') {
clientline[index] = c;
// are we too big for the buffer? start tossing out data
if (index >= BUFSIZ)
index = BUFSIZ -1;
// continue to read more data!
// got a \n or \r new line, which means the string is done
clientline[index] = 0;
// Print it out for debugging
// Look for substring such as a request to get the root file
if (strstr(clientline, "GET / ") != 0) {
// send a standard http response header
// print all the data files, use a helper to keep it clean
client.println("<h2>View data for the week of (dd-mm-yy):</h2>");
else if (strstr(clientline, "GET /") != 0) {
// this time no space after the /, so a sub-file!
char *filename;
filename = strtok(clientline + 5, "?"); // look after the "GET /" (5 chars) but before
// the "?" if a data file has been specified. A little trick, look for the " HTTP/1.1"
// string and turn the first character of the substring into a 0 to clear it out.
(strstr(clientline, " HTTP"))[0] = 0;
// print the file we want
File file =,FILE_READ);
if (!file) {
while(file.available()) {
int num_bytes_read;
uint8_t byte_buffer[32];,32);
else {
// everything else is a 404
// give the web browser time to receive the data
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.