Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
/* ************************************************************************
* *** Super Graphing Data Logger ***
* ************************************************************************
* Everett Robinson, December 2016. More at:
* This sketch relies on the SD and ethernet libraries in arduino 1.0 or newer.
* The following extra non standard libraries were also used, and will need to
* be added to the libraries folder:
* - Time:
* - EEPROMAnything:
* If this is your first time setting up this project, please go get the
* EEPROM_config sketch from so that you can
* configure the config struct in the EEPROM memory. Usage of the EEPROM
* is needed to make the project resiliant against a temporary loss of power.
* You must also ensure that you have the HC.htm file in the root directory
* of your SD card, as well as a data directory where the datafiles will be
* stored.
* This sketch combines the functionality of an existing fileserver example
* which can be found at
* with the Datalogger example that comes with the new SD library from 1.0,
* as well as some code from the UdpNtpClient example that cones with the
* ethernet library.
* Added to all of these are some tricks to make it manage and serve up the
* datafiles in conjunction with a page which uses highcharts JS to graph it.
* This is basically accomplished using the arduino by itself. Because I
* actually host the highcharts.js files externally, this is true more in
* theory than in actual practice, but oh well. It should work just fine to
* have the highcharts.js file on the arduino's SD card, though loading the
* page will be painfully slow.
* Some of the code this was derived from may or may not be under a GPL
* licence; I'm not entirely sure. I suppose anyone using this should treat
* it like it is too, but I don't really care too much.
* Also if one intends to use this for commercial applications, it may be
* necessary to purchase a license for Highcharts.
#include <SD.h>
#include <Ethernet.h>
#include <EthernetUdp.h>
#include <SPI.h>
#include <string.h>
#include <Time.h>
#include <EEPROM.h>
#include <EEPROMAnything.h>
#include <avr/pgmspace.h>
/************ ETHERNET STUFF ************/
byte mac[] = { 0x90, 0xA2, 0xDA, 0x00, 0x4C, 0x64 };
//byte ip[] = { 192,168,1, 100 };
EthernetServer server(80);
/************** NTP STUFF ***************/
unsigned int localPort = 8888; // local port to listen for UDP packets
IPAddress timeServer(128, 138, 140, 44);//NIST time server IP address: for more info
const int NTP_PACKET_SIZE= 48; //NTP time stamp is in the first 48 bytes of the message
byte packetBuffer[ NTP_PACKET_SIZE]; //buffer to hold incoming and outgoing packets
EthernetUDP Udp;
const int analogPin = 0;
unsigned long lastIntervalTime = 0; //The time the last measurement occured.
#define MEASURE_INTERVAL 600000 //10 minute intervals between measurements (in ms)
unsigned long newFileTime; //The time at which we should create a new week's file
#define FILE_INTERVAL 604800 //One week worth of seconds
//A structure that stores file config variables from EEPROM
typedef struct{
unsigned long newFileTime; //Keeps track of when a newfile should be made.
char workingFilename[19]; //The path and filename of the current week's file
} configuration;
configuration config; //Actually make our config struct
// Strings stored in flash mem for the Html Header (saves ram)
const char HeaderOK_0[] PROGMEM = "HTTP/1.1 200 OK"; //
const char HeaderOK_1[] PROGMEM = "Content-Type: text/html"; //
const char HeaderOK_2[] PROGMEM = ""; //
// A table of pointers to the flash memory strings for the header
const char* const HeaderOK_table[] PROGMEM = {
// A function for reasy printing of the headers
void HtmlHeaderOK(EthernetClient client) {
char buffer[30]; //A character array to hold the strings from the flash mem
for (int i = 0; i < 3; i++) {
strcpy_P(buffer, (char*)pgm_read_word(&(HeaderOK_table[i])));
client.println( buffer );
// Strings stored in flash mem for the Html 404 Header
const prog_char Header404_0[] PROGMEM = "HTTP/1.1 404 Not Found"; //
const prog_char Header404_1[] PROGMEM = "Content-Type: text/html"; //
const prog_char Header404_2[] PROGMEM = ""; //
const prog_char Header404_3[] PROGMEM = "<h2>File Not Found!</h2>";
// A table of pointers to the flash memory strings for the header
const char* const Header404_table[] PROGMEM = {
// Easy peasy 404 header function
void HtmlHeader404(EthernetClient client) {
char buffer[30]; //A character array to hold the strings from the flash mem
for (int i = 0; i < 4; i++) {
strcpy_P(buffer, (char*)pgm_read_word(&(Header404_table[i])));
client.println( buffer );
void setup() {
pinMode(10, OUTPUT); // set the SS pin as an output (necessary!)
digitalWrite(10, HIGH); // but turn off the W5100 chip!
// see if the card is present and can be initialized:
if (!SD.begin(4)) {
//Serial.println("Card failed, or not present");
// don't do anything more:
//Serial.println("card initialized.");
// The SD card is working, start the server and ethernet related stuff!
if (Ethernet.begin(mac) == 0) {
//Serial.println("Failed to configure Ethernet using DHCP");
// don't do anything more:
//Serial.println("DHCP configured!");
EEPROM_readAnything(0,config); // make sure our config struct is syncd with EEPROM
// A function that takes care of the listing of files for the
// main page one sees when they first connect to the arduino.
// it only lists the files in the /data/ folder. Make sure this
// exists on your SD card.
void ListFiles(EthernetClient client) {
File workingDir ="/data/");
while(true) {
File entry = workingDir.openNextFile();
if (! entry) {
client.print("<li><a href=\"/HC.htm?file=");
// A function to get the Ntp Time. This is used to make sure that the data
// points recorded by the arduino are referenced to some meaningful time
// which in our case is UTC represented as unix time (choosen because it
// works simply with highcharts without too much unecessary computation).
unsigned long getTime(){
sendNTPpacket(timeServer); // send an NTP packet to a time server
// wait to see if a reply is available
if ( Udp.parsePacket() ) {
// We've received a packet, read the data from it,NTP_PACKET_SIZE); // read the packet into the buffer
//the timestamp starts at byte 40 of the received packet and is four bytes,
// or two words, long. First, esxtract the two words:
unsigned long highWord = word(packetBuffer[40], packetBuffer[41]);
unsigned long lowWord = word(packetBuffer[42], packetBuffer[43]);
// combine the four bytes (two words) into a long integer
// this is NTP time (seconds since Jan 1 1900):
unsigned long secsSince1900 = highWord << 16 | lowWord;
// Unix time starts on Jan 1 1970. In seconds, that's 2208988800:
const unsigned long seventyYears = 2208988800UL;
// subtract seventy years:
unsigned long epoch = secsSince1900 - seventyYears;
// return Unix time:
return epoch;
// send an NTP request to the time server at the given address,
// necessary for getTime().
unsigned long sendNTPpacket(IPAddress& address){
// set all bytes in the buffer to 0
memset(packetBuffer, 0, NTP_PACKET_SIZE);
// Initialize values needed to form NTP request
// (see URL above for details on the packets)
packetBuffer[0] = 0b11100011; // LI, Version, Mode
packetBuffer[1] = 0; // Stratum, or type of clock
packetBuffer[2] = 6; // Polling Interval
packetBuffer[3] = 0xEC; // Peer Clock Precision
// 8 bytes of zero for Root Delay & Root Dispersion
packetBuffer[12] = 49;
packetBuffer[13] = 0x4E;
packetBuffer[14] = 49;
packetBuffer[15] = 52;
// all NTP fields have been given values, now
// you can send a packet requesting a timestamp:
Udp.beginPacket(address, 123); //NTP requests are to port 123
// How big our line buffer should be for sending the files over the ethernet.
// 75 has worked fine for me so far.
#define BUFSIZ 75
void loop(){
if ((millis() % lastIntervalTime) >= MEASURE_INTERVAL){ //Is it time for a new measurement?
char dataString[20] = "";
int count = 0;
unsigned long rawTime;
rawTime = getTime();
while((rawTime == 39) && (count < 12)){ //server seems to send 39 as an error code
delay(5000); //we want to retry if this happens. I chose
rawTime = getTime(); //12 retries because I'm stubborn/persistent.
count += 1; //NIST considers retry interval of <4s as DoS
} //attack, so fair warning.
if (rawTime != 39){ //If that worked, and we have a real time
//Decide if it's time to make a new file or not. Files are broken
//up like this to keep loading times for each chart bearable.
//Lots of string stuff happens to make a new filename if necessary.
if (rawTime >= config.newFileTime){
int dayInt = day(rawTime);
int monthInt = month(rawTime);
int yearInt = year(rawTime);
char newFilename[18] = "";
char dayStr[3];
char monthStr[3];
char yearStr[5];
char subYear[3];
if (dayInt < 10){
if (monthInt < 10){
//we only want the last two digits of the year
memcpy( subYear, &yearStr[2], 3 );
//make sure we update our config variables:
config.newFileTime += FILE_INTERVAL;
//Write the changes to EEPROM. Bad things may happen if power is lost midway through,
//but it's a small risk we take. Manual fix with EEPROM_config sketch can correct it.
EEPROM_writeAnything(0, config);
//get the values and setup the string we want to write to the file
int sensor = analogRead(analogPin);
char timeStr[12];
char sensorStr[6];
//open the file we'll be writing to.
File dataFile =, FILE_WRITE);
// if the file is available, write to it:
if (dataFile) {
// print to the serial port too:
// if the file isn't open, pop up an error:
else {
Serial.println("Error opening datafile for writing");
Serial.println("Couldn't resolve a time from the Ntp Server.");
//Update the time of the last measurment to the current timer value
lastIntervalTime = millis();
//No measurements to be made, make sure the webserver is available for connections.
char clientline[BUFSIZ];
int index = 0;
EthernetClient client = server.available();
if (client) {
// an http request ends with a blank line
boolean current_line_is_blank = true;
// reset the input buffer
index = 0;
while (client.connected()) {
if (client.available()) {
char c =;
// If it isn't a new line, add the character to the buffer
if (c != '\n' && c != '\r') {
clientline[index] = c;
// are we too big for the buffer? start tossing out data
if (index >= BUFSIZ)
index = BUFSIZ -1;
// continue to read more data!
// got a \n or \r new line, which means the string is done
clientline[index] = 0;
// Print it out for debugging
// Look for substring such as a request to get the root file
if (strstr(clientline, "GET / ") != 0) {
// send a standard http response header
// print all the data files, use a helper to keep it clean
client.println("<h2>View data for the week of (dd-mm-yy):</h2>");
else if (strstr(clientline, "GET /") != 0) {
// this time no space after the /, so a sub-file!
char *filename;
filename = strtok(clientline + 5, "?"); // look after the "GET /" (5 chars) but before
// the "?" if a data file has been specified. A little trick, look for the " HTTP/1.1"
// string and turn the first character of the substring into a 0 to clear it out.
(strstr(clientline, " HTTP"))[0] = 0;
// print the file we want
File file =,FILE_READ);
if (!file) {
while(file.available()) {
int num_bytes_read;
uint8_t byte_buffer[32];,32);
else {
// everything else is a 404
// give the web browser time to receive the data
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment